diff --git a/datastore/FertiScan/__init__.py b/datastore/FertiScan/__init__.py new file mode 100644 index 00000000..99667ddd --- /dev/null +++ b/datastore/FertiScan/__init__.py @@ -0,0 +1,49 @@ +import os +from dotenv import load_dotenv +import uuid + +load_dotenv() + +FERTISCAN_DB_URL = os.environ.get("FERTISCAN_DB_URL") +if FERTISCAN_DB_URL is None or FERTISCAN_DB_URL == "": + # raise ValueError("FERTISCAN_DB_URL is not set") + print("Warning: FERTISCAN_DB_URL not set") + +FERTISCAN_SCHEMA = os.environ.get("FERTISCAN_SCHEMA") +if FERTISCAN_SCHEMA is None or FERTISCAN_SCHEMA == "": + # raise ValueError("FERTISCAN_SCHEMA is not set") + print("Warning: FERTISCAN_SCHEMA not set") + +FERTISCAN_STORAGE_URL = os.environ.get("FERTISCAN_STORAGE_URL") +if FERTISCAN_STORAGE_URL is None or FERTISCAN_STORAGE_URL == "": + # raise ValueError("FERTISCAN_STORAGE_URL is not set") + print("Warning: FERTISCAN_STORAGE_URL not set") + + +async def register_analysis( + cursor, container_client, analysis_dict, picture_id: str, picture, folder="General" +): + """ + Register an analysis in the database + + Parameters: + - cursor: The cursor object to interact with the database. + - container_client: The container client of the user. + - analysis_dict (dict): The analysis to register in a dict string (soon to be json loaded). + - picture: The picture encoded to upload. + + Returns: + - The analysis_dict with the analysis_id added. + """ + try: + if picture_id is None or picture_id == "": + picture_id = str(uuid.uuid4()) + # if not azure_storage.is_a_folder(container_client, folder): + # azure_storage.create_folder(container_client, folder) + # azure_storage.upload_image(container_client, folder, picture, picture_id) + # analysis_id = analysis.new_analysis(cursor, json.dumps(analysis_dict)) + # analysis_dict["analysis_id"] = str(analysis_id) + return None + except Exception as e: + print(e.__str__()) + raise Exception("Datastore Unhandled Error") diff --git a/datastore/Nachet/__init__.py b/datastore/Nachet/__init__.py new file mode 100644 index 00000000..38df7ac0 --- /dev/null +++ b/datastore/Nachet/__init__.py @@ -0,0 +1,794 @@ +import os +from dotenv import load_dotenv +import datastore.db.queries.inference as inference +import datastore.db.queries.machine_learning as machine_learning +import datastore.db.metadata.machine_learning as ml_metadata +import datastore.db.metadata.inference as inference_metadata +import datastore.db.metadata.validator as validator +import datastore.db.queries.seed as seed +import datastore.db.queries.user as user +import datastore.db.queries.picture as picture +import datastore.db.metadata.picture_set as data_picture_set +import datastore.blob.azure_storage_api as azure_storage +import json +from datastore import ( + get_user_container_client, + BlobUploadError, + FolderCreationError, + UserNotOwnerError, +) + +load_dotenv() + +NACHET_BLOB_ACCOUNT = os.environ.get("NACHET_BLOB_ACCOUNT") +if NACHET_BLOB_ACCOUNT is None or NACHET_BLOB_ACCOUNT == "": + raise ValueError("NACHET_BLOB_ACCOUNT is not set") + +NACHET_BLOB_KEY = os.environ.get("NACHET_BLOB_KEY") +if NACHET_BLOB_KEY is None or NACHET_BLOB_KEY == "": + raise ValueError("NACHET_BLOB_KEY is not set") + +NACHET_STORAGE_URL = os.environ.get("NACHET_STORAGE_URL") +if NACHET_STORAGE_URL is None or NACHET_STORAGE_URL == "": + raise ValueError("NACHET_STORAGE_URL is not set") + +DEV_USER_EMAIL = os.environ.get("DEV_USER_EMAIL") +if DEV_USER_EMAIL is None or DEV_USER_EMAIL == "": + # raise ValueError("DEV_USER_EMAIL is not set") + print("Warning: DEV_USER_EMAIL not set") + +NACHET_DB_URL = os.environ.get("NACHET_DB_URL") +if NACHET_DB_URL is None or NACHET_DB_URL == "": + raise ValueError("NACHET_DB_URL is not set") + +NACHET_SCHEMA = os.environ.get("NACHET_SCHEMA") +if NACHET_SCHEMA is None or NACHET_SCHEMA == "": + raise ValueError("NACHET_SCHEMA is not set") + + +class InferenceCreationError(Exception): + pass + + +class InferenceFeedbackError(Exception): + pass + + +class MLRetrievalError(Exception): + pass + + +async def upload_picture_unknown( + cursor, user_id, picture_hash, container_client, picture_set_id=None +): + """ + Upload a picture that we don't know the seed to the user container + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - picture (str): The image to upload. + - container_client: The container client of the user. + """ + try: + + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + + empty_picture = json.dumps([]) + + default_picture_set = str(user.get_default_picture_set(cursor, user_id)) + if picture_set_id is None or str(picture_set_id) == default_picture_set: + picture_set_id = default_picture_set + folder_name = "General" + else: + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if folder_name is None: + folder_name = picture_set_id + + # Create picture instance in DB + picture_id = picture.new_picture_unknown( + cursor=cursor, + picture=empty_picture, + picture_set_id=picture_set_id, + ) + # Upload the picture to the Blob Storage + response = await azure_storage.upload_image( + container_client, folder_name, picture_set_id, picture_hash, picture_id + ) + # Update the picture metadata in the DB + data = { + "link": f"{folder_name}/" + str(picture_id), + "description": "Uploaded through the API", + } + + if not response: + raise BlobUploadError("Error uploading the picture") + + picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) + + return picture_id + except BlobUploadError or azure_storage.UploadImageError: + raise BlobUploadError("Error uploading the picture") + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") + + +async def upload_picture_known( + cursor, + user_id, + picture_hash, + container_client, + seed_id, + picture_set_id=None, + nb_seeds=None, + zoom_level=None, +): + """ + Upload a picture that the seed is known to the user container + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - picture (str): The image to upload. + - container_client: The container client of the user. + - seed_id: The UUID of the seed on the image. + - picture_set_id: The UUID of the picture set where to add the picture. + - nb_seeds: The number of seeds on the picture. + - zoom_level: The zoom level of the picture. + """ + try: + + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + + empty_picture = json.dumps([]) + # Create picture instance in DB + if picture_set_id is None: + picture_set_id = user.get_default_picture_set(cursor, user_id) + picture_id = picture.new_picture( + cursor=cursor, + picture=empty_picture, + picture_set_id=picture_set_id, + seed_id=seed_id, + ) + # Upload the picture to the Blob Storage + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if folder_name is None: + folder_name = picture_set_id + + response = await azure_storage.upload_image( + container_client, folder_name, picture_set_id, picture_hash, picture_id + ) + picture_link = ( + container_client.url + "/" + str(folder_name) + "/" + str(picture_id) + ) + # Create picture metadata and update DB instance (with link to Azure blob) + """ + data = picture_metadata.build_picture( + pic_encoded=picture_hash, + link=picture_link, + nb_seeds=nb_seeds, + zoom=zoom_level, + description="upload_picture_set script", + ) + """ + data = { + "link": picture_link, + "nb_seeds": nb_seeds, + "zoom": zoom_level, + "description": "Uploaded through the API", + } + if not response: + raise BlobUploadError("Error uploading the picture") + + picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) + + return picture_id + except BlobUploadError or azure_storage.UploadImageError: + raise BlobUploadError("Error uploading the picture") + except user.UserNotFoundError as e: + raise e + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") + + +async def upload_pictures( + cursor, + user_id, + picture_set_id, + container_client, + pictures, + seed_name: str, + zoom_level: float = None, + nb_seeds: int = None, +): + """ + Upload an array of pictures that the seed is known to the user container + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - picture (str): The image to upload. + - container_client: The container client of the user. + - pictures array: array of images to upload + - seed_name: The name of the seed on the images. + - picture_set_id: The UUID of the picture set where to add the pictures. + - nb_seeds: The number of seeds on the picture. + - zoom_level: The zoom level of the picture. + + Returns: + array of the new pictures UUID + """ + try: + + if not seed.is_seed_registered(cursor=cursor, seed_name=seed_name): + raise seed.SeedNotFoundError( + f"Seed not found based on the given name: {seed_name}" + ) + seed_id = seed.get_seed_id(cursor=cursor, seed_name=seed_name) + + pictures_id = [] + for picture_encoded in pictures: + id = await upload_picture_known( + cursor, + user_id, + picture_encoded, + container_client, + seed_id, + picture_set_id, + nb_seeds, + zoom_level, + ) + pictures_id.append(id) + + return pictures_id + except seed.SeedNotFoundError as e: + raise e + except user.UserNotFoundError as e: + raise e + except Exception: + raise BlobUploadError("An error occured during the upload of the pictures") + + +async def register_inference_result( + cursor, + user_id: str, + inference_dict, + picture_id: str, + pipeline_id: str, + type: int = 1, +): + """ + Register an inference result in the database + + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - inference (str): The inference to register in a dict string (soon to be json loaded). + - picture_id (str): The UUID of the picture. + - pipeline_id (str): The UUID of the pipeline. + + Returns: + - The inference_dict with the inference_id, box_id and top_id added. + """ + try: + trimmed_inference = inference_metadata.build_inference_import(inference_dict) + inference_id = inference.new_inference( + cursor, trimmed_inference, user_id, picture_id, type + ) + nb_object = int(inference_dict["totalBoxes"]) + inference_dict["inference_id"] = str(inference_id) + # loop through the boxes + for box_index in range(nb_object): + # TODO: adapt for multiple types of objects + if type == 1: + # TODO : adapt for the seed_id in the inference_dict + top_id = seed.get_seed_id( + cursor, inference_dict["boxes"][box_index]["label"] + ) + inference_dict["boxes"][box_index]["object_type_id"] = 1 + else: + raise inference.InferenceCreationError("Error: type not recognized") + box = inference_metadata.build_object_import( + inference_dict["boxes"][box_index] + ) + object_inference_id = inference.new_inference_object( + cursor, inference_id, box, type, False + ) + inference_dict["boxes"][box_index]["box_id"] = str(object_inference_id) + # loop through the topN Prediction + top_score = -1 + if "topN" in inference_dict["boxes"][box_index]: + for topN in inference_dict["boxes"][box_index]["topN"]: + + # Retrieve the right seed_id + seed_id = seed.get_seed_id(cursor, topN["label"]) + id = inference.new_seed_object( + cursor, seed_id, object_inference_id, topN["score"] + ) + topN["object_id"] = str(id) + if topN["score"] > top_score: + top_score = topN["score"] + top_id = id + else: + seed_id = seed.get_seed_id( + cursor, inference_dict["boxes"][box_index]["label"] + ) + top_id = inference.new_seed_object( + cursor, + seed_id, + object_inference_id, + inference_dict["boxes"][box_index]["score"], + ) + inference.set_inference_object_top_id(cursor, object_inference_id, top_id) + inference_dict["boxes"][box_index]["top_id"] = str(top_id) + + return inference_dict + except ValueError: + raise ValueError("The value of 'totalBoxes' is not an integer.") + except Exception as e: + print(e.__str__()) + raise Exception("Unhandled Error") + + +async def new_correction_inference_feedback(cursor, inference_dict, type: int = 1): + """ + TODO: doc + """ + try: + if "inference_id" in inference_dict.keys(): + inference_id = inference_dict["inference_id"] + else: + raise InferenceFeedbackError( + "Error: inference_id not found in the given infence_dict" + ) + if "user_id" in inference_dict.keys(): + user_id = inference_dict["user_id"] + if not (user.is_a_user_id(cursor, user_id)): + raise InferenceFeedbackError( + f"Error: user_id {user_id} not found in the database" + ) + else: + raise InferenceFeedbackError( + "Error: user_id not found in the given infence_dict" + ) + # if infence_dict["totalBoxes"] != len(inference_dict["boxes"] & infence_dict["totalBoxes"] > 0 ): + # if len(inference_dict["boxes"]) == 0: + # raise InferenceFeedbackError("Error: No boxes found in the given inference_dict") + # else if len(inference_dict["boxes"]) > infence_dict["totalBoxes"]: + # raise InferenceFeedbackError("Error: There are more boxes than the totalBoxes") + # else if len(inference_dict["boxes"]) < infence_dict["totalBoxes"]: + # raise InferenceFeedbackError("Error: There are less boxes than the totalBoxes") + if inference.is_inference_verified(cursor, inference_id): + raise InferenceFeedbackError( + f"Error: Inference {inference_id} is already verified" + ) + for object in inference_dict["boxes"]: + box_id = object["box_id"] + seed_name = object["label"] + seed_id = object["classId"] + # flag_seed = False + # flag_box_metadata = False + valid = False + box_metadata = object["box"] + + if box_id == "": + # This is a new box created by the user + + # Check if the seed is known + if seed_id == "" and seed_name == "": + raise InferenceFeedbackError( + "Error: seed_name and seed_id not found in the new box. We don't know what to do with it and this should not happen." + ) + if seed_id == "": + if seed.is_seed_registered(cursor, seed_name): + # Mistake from the FE, the seed is known in the database + seed_id = seed.get_seed_id(cursor, seed_name) + else: + # unknown seed + seed_id = seed.new_seed(cursor, seed_name) + # Create the new object + object_id = inference.new_inference_object( + cursor, inference_id, box_metadata, 1, True + ) + seed_object_id = inference.new_seed_object( + cursor, seed_id, object_id, 0 + ) + # Set the verified_id to the seed_object_id + inference.set_inference_object_verified_id( + cursor, object_id, seed_object_id + ) + valid = True + else: + if inference.is_object_verified(cursor, box_id): + raise InferenceFeedbackError( + f"Error: Object {box_id} is already verified" + ) + # This is a box that was created by the pipeline so it should be within the database + object_db = inference.get_inference_object(cursor, box_id) + object_metadata = object_db[1] + object_id = object_db[0] + + # Check if there are difference between the metadata + if not ( + inference_metadata.compare_object_metadata( + box_metadata, object_metadata["box"] + ) + ): + # Update the object metadata + # flag_box_metadata = True + inference.set_object_box_metadata( + cursor, box_id, json.dumps(box_metadata) + ) + + # Check if the seed is known + if seed_id == "": + if seed_name == "": + # box has been deleted by the user + valid = False + else: + valid = True + if seed.is_seed_registered(cursor, seed_name): + # The seed is known in the database and it was a mistake from the FE + seed_id = seed.get_seed_id(cursor, seed_name) + else: # The seed is not known in the database + seed_id = seed.new_seed(cursor, seed_name) + seed_object_id = inference.new_seed_object( + cursor, seed_id, object_id, 0 + ) + inference.set_inference_object_verified_id( + cursor, object_id, seed_object_id + ) + else: + # Box is still valid + valid = True + # Check if a new seed has been selected + top_inference_id = inference.get_inference_object_top_id( + cursor, object_db[0] + ) + new_top_id = inference.get_seed_object_id(cursor, seed_id, box_id) + + if new_top_id is None: + # Seed selected was not an inference guess, we need to create a new seed_object + new_top_id = inference.new_seed_object( + cursor, seed_id, box_id, 0 + ) + inference.set_inference_object_verified_id( + cursor, box_id, new_top_id + ) + # flag_seed = True + if top_inference_id != new_top_id: + # Seed was not correctly identified, set the verified_id to the correct seed_object.id + # flag_seed = True + inference.set_inference_object_verified_id( + cursor, box_id, new_top_id + ) + else: + # Seed was correctly identified, set the verified_id to the top_id + # flag_seed = False + inference.set_inference_object_verified_id( + cursor, box_id, top_inference_id + ) + + # Update the object validity + inference.set_inference_object_valid(cursor, box_id, valid) + inference.verify_inference_status(cursor, inference_id, user_id) + except InferenceFeedbackError: + raise + except Exception as e: + print(e.__str__()) + raise Exception("Datastore Unhandled Error") + + +async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id): + """ + Update objects when a perfect feedback is sent by a user and update the inference if all the objects in it are verified. + + Args: + cursor: The cursor object to interact with the database. + inference_id (str): id of the inference on which feedback is given + user_id (str): id of the user giving a feedback + boxes_id (str array): array of id of the objects that are correctly identified + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if boxes_id exists + for box_id in boxes_id: + if not inference.check_inference_object_exist(cursor, box_id): + raise inference.InferenceObjectNotFoundError( + f"Error: could not get inference object for id {box_id}" + ) + # Check if inference exists + if not inference.check_inference_exist(cursor, inference_id): + raise inference.InferenceNotFoundError( + f"Inference not found based on the given id: {inference_id}" + ) + + if inference.is_inference_verified(cursor, inference_id): + raise inference.InferenceAlreadyVerifiedError( + f"Can't add feedback to a verified inference, id: {inference_id}" + ) + + for object_id in boxes_id: + top_inference_id = inference.get_inference_object_top_id(cursor, object_id) + inference.set_inference_object_verified_id( + cursor, object_id, top_inference_id + ) + inference.set_inference_object_valid(cursor, object_id, True) + + inference.verify_inference_status(cursor, inference_id, user_id) + + except ( + user.UserNotFoundError, + inference.InferenceObjectNotFoundError, + inference.InferenceNotFoundError, + inference.InferenceAlreadyVerifiedError, + ) as e: + raise e + except Exception as e: + print(e) + raise Exception(f"Datastore Unhandled Error : {e}") + + +async def import_ml_structure_from_json_version(cursor, ml_version: dict): + """ + TODO: build tests + """ + pipelines = ml_version["pipelines"] + models = ml_version["models"] + # Create the models + for model in models: + model_db = ml_metadata.build_model_import(model) + task_id = machine_learning.get_task_id(cursor, model["task"]) + model_name = model["model_name"] + endpoint_name = model["endpoint_name"] + machine_learning.new_model(cursor, model_db, model_name, endpoint_name, task_id) + # Create the pipelines + for pipeline in pipelines: + pipeline_db = ml_metadata.build_pipeline_import(pipeline) + pipeline_name = pipeline["pipeline_name"] + model_ids = [] + # Establish the relationship between the pipelines and its models + for name_model in pipeline["models"]: + model_id = 0 + model_id = machine_learning.get_model_id_from_name(cursor, name_model) + if validator.is_valid_uuid(model_id): + model_ids.append(model_id) + else: + raise ValueError(f"Model {name_model} not found") + machine_learning.new_pipeline(cursor, pipeline_db, pipeline_name, model_ids) + + +async def get_ml_structure(cursor): + """ + This function retrieves the machine learning structure from the database. + + Returns a usable json object with the machine learning structure for the FE and BE + """ + try: + ml_structure = {"pipelines": [], "models": []} + pipelines = machine_learning.get_active_pipeline(cursor) + if len(pipelines) == 0: + raise MLRetrievalError("No Active pipelines found in the database.") + model_list = [] + for pipeline in pipelines: + # (id, name, active:bool, is_default: bool, data, model_ids: array) + pipeline_name = pipeline[1] + pipeline_id = pipeline[0] + default = pipeline[3] + model_ids = pipeline[5] + pipeline_dict = ml_metadata.build_pipeline_export( + pipeline[4], pipeline_name, pipeline_id, default, model_ids + ) + ml_structure["pipelines"].append(pipeline_dict) + for model_id in model_ids: + if model_id not in model_list: + model_list.append(model_id) + model_db = machine_learning.get_model(cursor, model_id) + # (id, name, endpoint_name, task_name, data,version: str) + model_name = model_db[1] + model_endpoint = model_db[2] + model_task = model_db[3] + model_version = model_db[5] + model_dict = ml_metadata.build_model_export( + model_db[4], + model_id, + model_name, + model_endpoint, + model_task, + model_version, + ) + ml_structure["models"].append(model_dict) + return ml_structure + except MLRetrievalError: + raise + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") + + +async def get_seed_info(cursor): + """ + This function retrieves the seed information from the database. + + Returns a usable json object with the seed information for the FE and BE + """ + seeds = seed.get_all_seeds(cursor) + seed_dict = {"seeds": []} + for seed_db in seeds: + seed_id = seed_db[0] + seed_name = seed_db[1] + seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) + return seed_dict + + +async def delete_picture_set_with_archive( + cursor, user_id, picture_set_id, container_client +): + """ + Delete a picture set from the database and the blob storage but archives inferences and pictures in dev container + + Args: + cursor: The cursor object to interact with the database. + user_id (str): id of the user + picture_set_id (str): id of the picture set to delete + container_client: The container client of the user. + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if picture set exists + if not picture.is_a_picture_set_id(cursor, picture_set_id): + raise picture.PictureSetNotFoundError( + f"Picture set not found based on the given id: {picture_set_id}" + ) + # Check user is owner of the picture set + if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + raise UserNotOwnerError( + f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" + ) + # Check if the picture set is the default picture set + general_folder_id = str(user.get_default_picture_set(cursor, user_id)) + if general_folder_id == picture_set_id: + raise picture.PictureSetDeleteError( + f"User can't delete the default picture set, user uuid :{user_id}" + ) + + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if folder_name is None: + folder_name = picture_set_id + validated_pictures = picture.get_validated_pictures(cursor, picture_set_id) + + dev_user_id = user.get_user_id(cursor, DEV_USER_EMAIL) + dev_container_client = await get_user_container_client( + dev_user_id, NACHET_STORAGE_URL, NACHET_BLOB_ACCOUNT, NACHET_BLOB_KEY + ) + if not dev_container_client.exists(): + raise BlobUploadError( + f"Error while connecting to the dev container: {dev_user_id}" + ) + if not await azure_storage.is_a_folder(dev_container_client, str(user_id)): + await azure_storage.create_folder(dev_container_client, str(user_id)) + + picture_set = data_picture_set.build_picture_set( + dev_user_id, len(validated_pictures) + ) + dev_picture_set_id = picture.new_picture_set( + cursor=cursor, + picture_set=picture_set, + user_id=dev_user_id, + folder_name=folder_name, + ) + + folder_created = await azure_storage.create_dev_container_folder( + dev_container_client, str(picture_set_id), folder_name, user_id + ) + if not folder_created: + raise FolderCreationError( + f"Error while creating this folder : {picture_set_id}" + ) + + for picture_id in picture.get_validated_pictures(cursor, picture_set_id): + picture_metadata = picture.get_picture(cursor, picture_id) + blob_name = f"{folder_name}/{str(picture_id)}.png" + # change the link in the metadata + picture_metadata["link"] = f"{user_id}/{folder_name}/{picture_id}" + picture.update_picture_metadata( + cursor, picture_id, json.dumps(picture_metadata), 0 + ) + # set picture set to dev one + picture.update_picture_picture_set_id( + cursor, picture_id, dev_picture_set_id + ) + # move the picture to the dev container + new_blob_name = "{}/{}/{}.png".format(user_id, folder_name, str(picture_id)) + if not ( + await azure_storage.move_blob( + blob_name, + new_blob_name, + dev_picture_set_id, + container_client, + dev_container_client, + ) + ): + raise BlobUploadError( + f"Error while moving the picture : {picture_id} to the dev container" + ) + + if len(picture.get_validated_pictures(cursor, picture_set_id)) > 0: + raise picture.PictureSetDeleteError( + f"Can't delete the folder, there are still validated pictures in it, folder name : {picture_set_id}" + ) + + # Delete the folder in the blob storage + await azure_storage.delete_folder(container_client, picture_set_id) + # Delete the picture set + picture.delete_picture_set(cursor, picture_set_id) + + return dev_picture_set_id + except ( + user.UserNotFoundError, + picture.PictureSetNotFoundError, + picture.PictureSetDeleteError, + UserNotOwnerError, + ) as e: + raise e + except Exception as e: + print(f"Datastore Unhandled Error : {e}") + raise Exception("Datastore Unhandled Error") + + +async def find_validated_pictures(cursor, user_id, picture_set_id): + """ + Find pictures that have been validated by the user in the given picture set + + Args: + cursor: The cursor object to interact with the database. + user_id (str): id of the user that should be the owner of the picture set + picture_set_id (str): id of the picture set + + Returns: + list of picture_id + """ + try: + # Check if user exists + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + # Check if picture set exists + if not picture.is_a_picture_set_id(cursor, picture_set_id): + raise picture.PictureSetNotFoundError( + f"Picture set not found based on the given id: {picture_set_id}" + ) + # Check user is owner of the picture set + if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: + raise UserNotOwnerError( + f"User isn't owner of this folder, user uuid :{user_id}, folder uuid : {picture_set_id}" + ) + + validated_pictures_id = picture.get_validated_pictures(cursor, picture_set_id) + return validated_pictures_id + except ( + user.UserNotFoundError, + picture.PictureSetNotFoundError, + UserNotOwnerError, + ): + raise + except Exception as e: + print(e) + raise Exception("Datastore Unhandled Error") diff --git a/datastore/__init__.py b/datastore/__init__.py index 7f92c5e5..75e3d0f4 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -3,47 +3,17 @@ and the user container in the blob storage. """ +import json import datastore.db.queries.user as user -import datastore.db.queries.inference as inference -import datastore.db.queries.machine_learning as machine_learning import datastore.db.queries.picture as picture -import datastore.db.metadata.machine_learning as ml_metadata -import datastore.db.metadata.inference as inference_metadata -import datastore.db.metadata.validator as validator -import datastore.db.queries.seed as seed import datastore.db.metadata.picture_set as data_picture_set import datastore.blob as blob import datastore.blob.azure_storage_api as azure_storage -import uuid -import json from azure.storage.blob import BlobServiceClient, ContainerClient -import os from dotenv import load_dotenv load_dotenv() -NACHET_BLOB_ACCOUNT = os.environ.get("NACHET_BLOB_ACCOUNT") -if NACHET_BLOB_ACCOUNT is None or NACHET_BLOB_ACCOUNT == "": - raise ValueError("NACHET_BLOB_ACCOUNT is not set") - -NACHET_BLOB_KEY = os.environ.get("NACHET_BLOB_KEY") -if NACHET_BLOB_KEY is None or NACHET_BLOB_KEY == "": - raise ValueError("NACHET_BLOB_KEY is not set") - -NACHET_STORAGE_URL = os.environ.get("NACHET_STORAGE_URL") -if NACHET_STORAGE_URL is None or NACHET_STORAGE_URL == "": - raise ValueError("NACHET_STORAGE_URL is not set") - -FERTISCAN_STORAGE_URL = os.environ.get("FERTISCAN_STORAGE_URL") -if FERTISCAN_STORAGE_URL is None or FERTISCAN_STORAGE_URL == "": - # raise ValueError("FERTISCAN_STORAGE_URL is not set") - print("Warning: FERTISCAN_STORAGE_URL not set") - -DEV_USER_EMAIL = os.environ.get("DEV_USER_EMAIL") -if DEV_USER_EMAIL is None or DEV_USER_EMAIL == "": - # raise ValueError("DEV_USER_EMAIL is not set") - print("Warning: DEV_USER_EMAIL not set") - class UserAlreadyExistsError(Exception): pass @@ -53,10 +23,6 @@ class UserNotOwnerError(Exception): pass -class MLRetrievalError(Exception): - pass - - class BlobUploadError(Exception): pass @@ -69,14 +35,6 @@ class FolderCreationError(Exception): pass -class InferenceCreationError(Exception): - pass - - -class InferenceFeedbackError(Exception): - pass - - class User: def __init__(self, email: str, id: str = None, tier: str = "user"): self.id = id @@ -155,9 +113,7 @@ async def new_user(cursor, email, connection_string, tier="user") -> User: raise Exception("Datastore Unhandled Error") -async def get_user_container_client( - user_id, tier="user", storage_url=NACHET_STORAGE_URL -): +async def get_user_container_client(user_id, storage_url, account, key, tier="user"): """ Get the container client of a user @@ -166,7 +122,7 @@ async def get_user_container_client( Returns: ContainerClient object """ - sas = blob.get_account_sas(NACHET_BLOB_ACCOUNT, NACHET_BLOB_KEY) + sas = blob.get_account_sas(account, key) # Get the container client container_client = await azure_storage.mount_container( storage_url, user_id, True, tier, sas @@ -228,581 +184,6 @@ async def create_picture_set( raise BlobUploadError("An error occured during the upload of the picture set") -async def upload_picture_unknown( - cursor, user_id, picture_hash, container_client, picture_set_id=None -): - """ - Upload a picture that we don't know the seed to the user container - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - picture (str): The image to upload. - - container_client: The container client of the user. - """ - try: - - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - - empty_picture = json.dumps([]) - - default_picture_set = str(user.get_default_picture_set(cursor, user_id)) - if picture_set_id is None or str(picture_set_id) == default_picture_set: - picture_set_id = default_picture_set - folder_name = "General" - else: - folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None: - folder_name = picture_set_id - - # Create picture instance in DB - picture_id = picture.new_picture_unknown( - cursor=cursor, - picture=empty_picture, - picture_set_id=picture_set_id, - ) - # Upload the picture to the Blob Storage - response = await azure_storage.upload_image( - container_client, folder_name, picture_set_id, picture_hash, picture_id - ) - # Update the picture metadata in the DB - data = { - "link": f"{folder_name}/" + str(picture_id), - "description": "Uploaded through the API", - } - - if not response: - raise BlobUploadError("Error uploading the picture") - - picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) - - return picture_id - except BlobUploadError or azure_storage.UploadImageError: - raise BlobUploadError("Error uploading the picture") - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def upload_picture_known( - cursor, - user_id, - picture_hash, - container_client, - seed_id, - picture_set_id=None, - nb_seeds=None, - zoom_level=None, -): - """ - Upload a picture that the seed is known to the user container - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - picture (str): The image to upload. - - container_client: The container client of the user. - - seed_id: The UUID of the seed on the image. - - picture_set_id: The UUID of the picture set where to add the picture. - - nb_seeds: The number of seeds on the picture. - - zoom_level: The zoom level of the picture. - """ - try: - - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - - empty_picture = json.dumps([]) - # Create picture instance in DB - if picture_set_id is None: - picture_set_id = user.get_default_picture_set(cursor, user_id) - picture_id = picture.new_picture( - cursor=cursor, - picture=empty_picture, - picture_set_id=picture_set_id, - seed_id=seed_id, - ) - # Upload the picture to the Blob Storage - folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None: - folder_name = picture_set_id - - response = await azure_storage.upload_image( - container_client, folder_name, picture_set_id, picture_hash, picture_id - ) - picture_link = ( - container_client.url + "/" + str(folder_name) + "/" + str(picture_id) - ) - # Create picture metadata and update DB instance (with link to Azure blob) - """ - data = picture_metadata.build_picture( - pic_encoded=picture_hash, - link=picture_link, - nb_seeds=nb_seeds, - zoom=zoom_level, - description="upload_picture_set script", - ) - """ - data = { - "link": picture_link, - "nb_seeds": nb_seeds, - "zoom": zoom_level, - "description": "Uploaded through the API", - } - if not response: - raise BlobUploadError("Error uploading the picture") - - picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) - - return picture_id - except BlobUploadError or azure_storage.UploadImageError: - raise BlobUploadError("Error uploading the picture") - except user.UserNotFoundError as e: - raise e - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def upload_pictures( - cursor, - user_id, - picture_set_id, - container_client, - pictures, - seed_name: str, - zoom_level: float = None, - nb_seeds: int = None, -): - """ - Upload an array of pictures that the seed is known to the user container - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - picture (str): The image to upload. - - container_client: The container client of the user. - - pictures array: array of images to upload - - seed_name: The name of the seed on the images. - - picture_set_id: The UUID of the picture set where to add the pictures. - - nb_seeds: The number of seeds on the picture. - - zoom_level: The zoom level of the picture. - - Returns: - array of the new pictures UUID - """ - try: - - if not seed.is_seed_registered(cursor=cursor, seed_name=seed_name): - raise seed.SeedNotFoundError( - f"Seed not found based on the given name: {seed_name}" - ) - seed_id = seed.get_seed_id(cursor=cursor, seed_name=seed_name) - - pictures_id = [] - for picture_encoded in pictures: - id = await upload_picture_known( - cursor, - user_id, - picture_encoded, - container_client, - seed_id, - picture_set_id, - nb_seeds, - zoom_level, - ) - pictures_id.append(id) - - return pictures_id - except seed.SeedNotFoundError as e: - raise e - except user.UserNotFoundError as e: - raise e - except Exception: - raise BlobUploadError("An error occured during the upload of the pictures") - - -async def register_inference_result( - cursor, - user_id: str, - inference_dict, - picture_id: str, - pipeline_id: str, - type: int = 1, -): - """ - Register an inference result in the database - - Parameters: - - cursor: The cursor object to interact with the database. - - user_id (str): The UUID of the user. - - inference (str): The inference to register in a dict string (soon to be json loaded). - - picture_id (str): The UUID of the picture. - - pipeline_id (str): The UUID of the pipeline. - - Returns: - - The inference_dict with the inference_id, box_id and top_id added. - """ - try: - trimmed_inference = inference_metadata.build_inference_import(inference_dict) - inference_id = inference.new_inference( - cursor, trimmed_inference, user_id, picture_id, type - ) - nb_object = int(inference_dict["totalBoxes"]) - inference_dict["inference_id"] = str(inference_id) - # loop through the boxes - for box_index in range(nb_object): - # TODO: adapt for multiple types of objects - if type == 1: - # TODO : adapt for the seed_id in the inference_dict - top_id = seed.get_seed_id( - cursor, inference_dict["boxes"][box_index]["label"] - ) - inference_dict["boxes"][box_index]["object_type_id"] = 1 - else: - raise inference.InferenceCreationError("Error: type not recognized") - box = inference_metadata.build_object_import( - inference_dict["boxes"][box_index] - ) - object_inference_id = inference.new_inference_object( - cursor, inference_id, box, type, False - ) - inference_dict["boxes"][box_index]["box_id"] = str(object_inference_id) - # loop through the topN Prediction - top_score = -1 - if "topN" in inference_dict["boxes"][box_index]: - for topN in inference_dict["boxes"][box_index]["topN"]: - - # Retrieve the right seed_id - seed_id = seed.get_seed_id(cursor, topN["label"]) - id = inference.new_seed_object( - cursor, seed_id, object_inference_id, topN["score"] - ) - topN["object_id"] = str(id) - if topN["score"] > top_score: - top_score = topN["score"] - top_id = id - else: - seed_id = seed.get_seed_id( - cursor, inference_dict["boxes"][box_index]["label"] - ) - top_id = inference.new_seed_object( - cursor, - seed_id, - object_inference_id, - inference_dict["boxes"][box_index]["score"], - ) - inference.set_inference_object_top_id(cursor, object_inference_id, top_id) - inference_dict["boxes"][box_index]["top_id"] = str(top_id) - - return inference_dict - except ValueError: - raise ValueError("The value of 'totalBoxes' is not an integer.") - except Exception as e: - print(e.__str__()) - raise Exception("Unhandled Error") - - -async def new_correction_inference_feedback(cursor, inference_dict, type: int = 1): - """ - TODO: doc - """ - try: - if "inferenceId" in inference_dict.keys(): - inference_id = inference_dict["inferenceId"] - else: - raise InferenceFeedbackError( - "Error: inference_id not found in the given infence_dict" - ) - if "userId" in inference_dict.keys(): - user_id = inference_dict["userId"] - if not (user.is_a_user_id(cursor, user_id)): - raise InferenceFeedbackError( - f"Error: user_id {user_id} not found in the database" - ) - else: - raise InferenceFeedbackError( - "Error: user_id not found in the given infence_dict" - ) - # if infence_dict["totalBoxes"] != len(inference_dict["boxes"] & infence_dict["totalBoxes"] > 0 ): - # if len(inference_dict["boxes"]) == 0: - # raise InferenceFeedbackError("Error: No boxes found in the given inference_dict") - # else if len(inference_dict["boxes"]) > infence_dict["totalBoxes"]: - # raise InferenceFeedbackError("Error: There are more boxes than the totalBoxes") - # else if len(inference_dict["boxes"]) < infence_dict["totalBoxes"]: - # raise InferenceFeedbackError("Error: There are less boxes than the totalBoxes") - if inference.is_inference_verified(cursor, inference_id): - raise InferenceFeedbackError( - f"Error: Inference {inference_id} is already verified" - ) - for object in inference_dict["boxes"]: - box_id = object["boxId"] - seed_name = object["label"] - seed_id = object["classId"] - # flag_seed = False - # flag_box_metadata = False - valid = False - box_metadata = object["box"] - - if box_id == "": - # This is a new box created by the user - - # Check if the seed is known - if seed_id == "" and seed_name == "": - raise InferenceFeedbackError( - "Error: seed_name and seed_id not found in the new box. We don't know what to do with it and this should not happen." - ) - if seed_id == "": - if seed.is_seed_registered(cursor, seed_name): - # Mistake from the FE, the seed is known in the database - seed_id = seed.get_seed_id(cursor, seed_name) - else: - # unknown seed - seed_id = seed.new_seed(cursor, seed_name) - # Create the new object - object_id = inference.new_inference_object( - cursor, inference_id, box_metadata, 1, True - ) - seed_object_id = inference.new_seed_object( - cursor, seed_id, object_id, 0 - ) - # Set the verified_id to the seed_object_id - inference.set_inference_object_verified_id( - cursor, object_id, seed_object_id - ) - valid = True - else: - if inference.is_object_verified(cursor, box_id): - raise InferenceFeedbackError( - f"Error: Object {box_id} is already verified" - ) - # This is a box that was created by the pipeline so it should be within the database - object_db = inference.get_inference_object(cursor, box_id) - object_metadata = object_db[1] - object_id = object_db[0] - - # Check if there are difference between the metadata - if not ( - inference_metadata.compare_object_metadata( - box_metadata, object_metadata["box"] - ) - ): - # Update the object metadata - # flag_box_metadata = True - inference.set_object_box_metadata( - cursor, box_id, json.dumps(box_metadata) - ) - - # Check if the seed is known - if seed_id == "": - if seed_name == "": - # box has been deleted by the user - valid = False - else: - valid = True - if seed.is_seed_registered(cursor, seed_name): - # The seed is known in the database and it was a mistake from the FE - seed_id = seed.get_seed_id(cursor, seed_name) - else: # The seed is not known in the database - seed_id = seed.new_seed(cursor, seed_name) - seed_object_id = inference.new_seed_object( - cursor, seed_id, object_id, 0 - ) - inference.set_inference_object_verified_id( - cursor, object_id, seed_object_id - ) - else: - # Box is still valid - valid = True - # Check if a new seed has been selected - top_inference_id = inference.get_inference_object_top_id( - cursor, object_db[0] - ) - new_top_id = inference.get_seed_object_id(cursor, seed_id, box_id) - - if new_top_id is None: - # Seed selected was not an inference guess, we need to create a new seed_object - new_top_id = inference.new_seed_object( - cursor, seed_id, box_id, 0 - ) - inference.set_inference_object_verified_id( - cursor, box_id, new_top_id - ) - # flag_seed = True - if top_inference_id != new_top_id: - # Seed was not correctly identified, set the verified_id to the correct seed_object.id - # flag_seed = True - inference.set_inference_object_verified_id( - cursor, box_id, new_top_id - ) - else: - # Seed was correctly identified, set the verified_id to the top_id - # flag_seed = False - inference.set_inference_object_verified_id( - cursor, box_id, top_inference_id - ) - - # Update the object validity - inference.set_inference_object_valid(cursor, box_id, valid) - inference.verify_inference_status(cursor, inference_id, user_id) - except InferenceFeedbackError: - raise - except Exception as e: - print(e.__str__()) - raise Exception("Datastore Unhandled Error") - - -async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id): - """ - Update objects when a perfect feedback is sent by a user and update the inference if all the objects in it are verified. - - Args: - cursor: The cursor object to interact with the database. - inference_id (str): id of the inference on which feedback is given - user_id (str): id of the user giving a feedback - boxes_id (str array): array of id of the objects that are correctly identified - """ - try: - # Check if user exists - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - # Check if boxes_id exists - for box_id in boxes_id: - if not inference.check_inference_object_exist(cursor, box_id): - raise inference.InferenceObjectNotFoundError( - f"Error: could not get inference object for id {box_id}" - ) - # Check if inference exists - if not inference.check_inference_exist(cursor, inference_id): - raise inference.InferenceNotFoundError( - f"Inference not found based on the given id: {inference_id}" - ) - - if inference.is_inference_verified(cursor, inference_id): - raise inference.InferenceAlreadyVerifiedError( - f"Can't add feedback to a verified inference, id: {inference_id}" - ) - - for object_id in boxes_id: - top_inference_id = inference.get_inference_object_top_id(cursor, object_id) - inference.set_inference_object_verified_id( - cursor, object_id, top_inference_id - ) - inference.set_inference_object_valid(cursor, object_id, True) - - inference.verify_inference_status(cursor, inference_id, user_id) - - except ( - user.UserNotFoundError, - inference.InferenceObjectNotFoundError, - inference.InferenceNotFoundError, - inference.InferenceAlreadyVerifiedError, - ) as e: - raise e - except Exception as e: - print(e) - raise Exception(f"Datastore Unhandled Error : {e}") - - -async def import_ml_structure_from_json_version(cursor, ml_version: dict): - """ - TODO: build tests - """ - pipelines = ml_version["pipelines"] - models = ml_version["models"] - # Create the models - for model in models: - model_db = ml_metadata.build_model_import(model) - task_id = machine_learning.get_task_id(cursor, model["task"]) - model_name = model["model_name"] - endpoint_name = model["endpoint_name"] - machine_learning.new_model(cursor, model_db, model_name, endpoint_name, task_id) - # Create the pipelines - for pipeline in pipelines: - pipeline_db = ml_metadata.build_pipeline_import(pipeline) - pipeline_name = pipeline["pipeline_name"] - model_ids = [] - # Establish the relationship between the pipelines and its models - for name_model in pipeline["models"]: - model_id = 0 - model_id = machine_learning.get_model_id_from_name(cursor, name_model) - if validator.is_valid_uuid(model_id): - model_ids.append(model_id) - else: - raise ValueError(f"Model {name_model} not found") - machine_learning.new_pipeline(cursor, pipeline_db, pipeline_name, model_ids) - - -async def get_ml_structure(cursor): - """ - This function retrieves the machine learning structure from the database. - - Returns a usable json object with the machine learning structure for the FE and BE - """ - try: - ml_structure = {"pipelines": [], "models": []} - pipelines = machine_learning.get_active_pipeline(cursor) - if len(pipelines) == 0: - raise MLRetrievalError("No Active pipelines found in the database.") - model_list = [] - for pipeline in pipelines: - # (id, name, active:bool, is_default: bool, data, model_ids: array) - pipeline_name = pipeline[1] - pipeline_id = pipeline[0] - default = pipeline[3] - model_ids = pipeline[5] - pipeline_dict = ml_metadata.build_pipeline_export( - pipeline[4], pipeline_name, pipeline_id, default, model_ids - ) - ml_structure["pipelines"].append(pipeline_dict) - for model_id in model_ids: - if model_id not in model_list: - model_list.append(model_id) - model_db = machine_learning.get_model(cursor, model_id) - # (id, name, endpoint_name, task_name, data,version: str) - model_name = model_db[1] - model_endpoint = model_db[2] - model_task = model_db[3] - model_version = model_db[5] - model_dict = ml_metadata.build_model_export( - model_db[4], - model_id, - model_name, - model_endpoint, - model_task, - model_version, - ) - ml_structure["models"].append(model_dict) - return ml_structure - except MLRetrievalError: - raise - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def get_seed_info(cursor): - """ - This function retrieves the seed information from the database. - - Returns a usable json object with the seed information for the FE and BE - """ - seeds = seed.get_all_seeds(cursor) - seed_dict = {"seeds": []} - for seed_db in seeds: - seed_id = seed_db[0] - seed_name = seed_db[1] - seed_dict["seeds"].append({"seed_id": seed_id, "seed_name": seed_name}) - return seed_dict - - async def get_picture_sets_info(cursor, user_id: str): """This function retrieves the picture sets names and number of pictures from the database. @@ -876,181 +257,64 @@ async def delete_picture_set_permanently( raise Exception("Datastore Unhandled Error") -async def delete_picture_set_with_archive( - cursor, user_id, picture_set_id, container_client +async def upload_pictures( + cursor, user_id, hashed_pictures, container_client, picture_set_id=None ): """ - Delete a picture set from the database and the blob storage but archives inferences and pictures in dev container + Upload a picture that we don't know the seed to the user container - Args: - cursor: The cursor object to interact with the database. - user_id (str): id of the user - picture_set_id (str): id of the picture set to delete - container_client: The container client of the user. + Parameters: + - cursor: The cursor object to interact with the database. + - user_id (str): The UUID of the user. + - hashed_pictures ([str]): The images to upload. + - container_client: The container client of the user. """ try: - # Check if user exists - if not user.is_a_user_id(cursor=cursor, user_id=user_id): - raise user.UserNotFoundError( - f"User not found based on the given id: {user_id}" - ) - # Check if picture set exists - if not picture.is_a_picture_set_id(cursor, picture_set_id): - raise picture.PictureSetNotFoundError( - f"Picture set not found based on the given id: {picture_set_id}" - ) - # Check user is owner of the picture set - if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: - raise UserNotOwnerError( - f"User can't delete this folder, user uuid :{user_id}, folder name : {picture_set_id}" - ) - # Check if the picture set is the default picture set - general_folder_id = str(user.get_default_picture_set(cursor, user_id)) - if general_folder_id == picture_set_id: - raise picture.PictureSetDeleteError( - f"User can't delete the default picture set, user uuid :{user_id}" - ) - - folder_name = picture.get_picture_set_name(cursor, picture_set_id) - if folder_name is None: - folder_name = picture_set_id - validated_pictures = picture.get_validated_pictures(cursor, picture_set_id) - - dev_user_id = user.get_user_id(cursor, DEV_USER_EMAIL) - dev_container_client = await get_user_container_client(dev_user_id) - - if not await azure_storage.is_a_folder(dev_container_client, str(user_id)): - await azure_storage.create_folder(dev_container_client, str(user_id)) - - picture_set = data_picture_set.build_picture_set( - dev_user_id, len(validated_pictures) - ) - dev_picture_set_id = picture.new_picture_set( - cursor=cursor, - picture_set=picture_set, - user_id=dev_user_id, - folder_name=folder_name, - ) - - folder_created = await azure_storage.create_dev_container_folder( - dev_container_client, str(picture_set_id), folder_name, user_id - ) - if not folder_created: - raise FolderCreationError( - f"Error while creating this folder : {picture_set_id}" - ) - - for picture_id in picture.get_validated_pictures(cursor, picture_set_id): - picture_metadata = picture.get_picture(cursor, picture_id) - blob_name = f"{folder_name}/{str(picture_id)}.png" - # change the link in the metadata - picture_metadata["link"] = f"{user_id}/{folder_name}/{picture_id}" - picture.update_picture_metadata( - cursor, picture_id, json.dumps(picture_metadata), 0 - ) - # set picture set to dev one - picture.update_picture_picture_set_id( - cursor, picture_id, dev_picture_set_id - ) - # move the picture to the dev container - new_blob_name = "{}/{}/{}.png".format(user_id, folder_name, picture_id) - await azure_storage.move_blob( - blob_name, - new_blob_name, - dev_picture_set_id, - container_client, - dev_container_client, - ) - - if len(picture.get_validated_pictures(cursor, picture_set_id)) > 0: - raise picture.PictureSetDeleteError( - f"Can't delete the folder, there are still validated pictures in it, folder name : {picture_set_id}" - ) - - # Delete the folder in the blob storage - await azure_storage.delete_folder(container_client, picture_set_id) - # Delete the picture set - picture.delete_picture_set(cursor, picture_set_id) - - return dev_picture_set_id - except ( - user.UserNotFoundError, - picture.PictureSetNotFoundError, - picture.PictureSetDeleteError, - UserNotOwnerError, - ) as e: - raise e - except Exception as e: - print(f"Datastore Unhandled Error : {e}") - raise Exception("Datastore Unhandled Error") - - -async def find_validated_pictures(cursor, user_id, picture_set_id): - """ - Find pictures that have been validated by the user in the given picture set - Args: - cursor: The cursor object to interact with the database. - user_id (str): id of the user that should be the owner of the picture set - picture_set_id (str): id of the picture set - - Returns: - list of picture_id - """ - try: - # Check if user exists if not user.is_a_user_id(cursor=cursor, user_id=user_id): raise user.UserNotFoundError( f"User not found based on the given id: {user_id}" ) - # Check if picture set exists - if not picture.is_a_picture_set_id(cursor, picture_set_id): - raise picture.PictureSetNotFoundError( - f"Picture set not found based on the given id: {picture_set_id}" - ) - # Check user is owner of the picture set - if picture.get_picture_set_owner_id(cursor, picture_set_id) != user_id: - raise UserNotOwnerError( - f"User isn't owner of this folder, user uuid :{user_id}, folder uuid : {picture_set_id}" - ) - - validated_pictures_id = picture.get_validated_pictures(cursor, picture_set_id) - return validated_pictures_id - except ( - user.UserNotFoundError, - picture.PictureSetNotFoundError, - UserNotOwnerError, - ) as e: - raise e - except Exception as e: - print(e) - raise Exception("Datastore Unhandled Error") - - -async def register_analysis( - cursor, container_client, analysis_dict, picture_id: str, picture, folder="General" -): - """ - Register an analysis in the database - Parameters: - - cursor: The cursor object to interact with the database. - - container_client: The container client of the user. - - analysis_dict (dict): The analysis to register in a dict string (soon to be json loaded). - - picture: The picture encoded to upload. + empty_picture = json.dumps([]) - Returns: - - The analysis_dict with the analysis_id added. - """ - try: - if picture_id is None or picture_id == "": - picture_id = str(uuid.uuid4()) - # if not azure_storage.is_a_folder(container_client, folder): - # azure_storage.create_folder(container_client, folder) - # azure_storage.upload_image(container_client, folder, picture, picture_id) - # analysis_id = analysis.new_analysis(cursor, json.dumps(analysis_dict)) - #analysis_dict["analysis_id"] = str(analysis_id) - return None + default_picture_set = str(user.get_default_picture_set(cursor, user_id)) + if picture_set_id is None or str(picture_set_id) == default_picture_set: + picture_set_id = default_picture_set + folder_name = "General" + else: + folder_name = picture.get_picture_set_name(cursor, picture_set_id) + if folder_name is None: + folder_name = picture_set_id + pic_ids = [] + for picture_hash in hashed_pictures: + # Create picture instance in DB + picture_id = picture.new_picture_unknown( + cursor=cursor, + picture=empty_picture, + nb_objects=len(hashed_pictures), + picture_set_id=picture_set_id, + ) + # Upload the picture to the Blob Storage + response = await azure_storage.upload_image( + container_client, folder_name, picture_set_id, picture_hash, picture_id + ) + # Update the picture metadata in the DB + data = { + "link": f"{folder_name}/" + str(picture_id), + "description": "Uploaded through the API", + } + + if not response: + raise BlobUploadError("Error uploading the picture") + + picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) + pic_ids.append(picture_id) + return pic_ids + except BlobUploadError or azure_storage.UploadImageError: + raise BlobUploadError("Error uploading the picture") + except user.UserNotFoundError: + raise except Exception as e: - print(e.__str__()) + print(e) raise Exception("Datastore Unhandled Error") diff --git a/datastore/db/__init__.py b/datastore/db/__init__.py index 3479f885..a43a64ed 100644 --- a/datastore/db/__init__.py +++ b/datastore/db/__init__.py @@ -1,43 +1,14 @@ """ This module contains the function interacting with the database directly. """ -import os + import psycopg from dotenv import load_dotenv load_dotenv() -NACHET_DB_URL = os.environ.get("NACHET_DB_URL") -if NACHET_DB_URL is None or NACHET_DB_URL == "": - raise ValueError("NACHET_DB_URL is not set") - -NACHET_SCHEMA = os.environ.get("NACHET_SCHEMA") -if NACHET_SCHEMA is None or NACHET_SCHEMA == "": - raise ValueError("NACHET_SCHEMA is not set") - -FERTISCAN_DB_URL = os.environ.get("FERTISCAN_DB_URL") -if FERTISCAN_DB_URL is None or FERTISCAN_DB_URL == "": - # raise ValueError("FERTISCAN_DB_URL is not set") - print("Warning: FERTISCAN_DB_URL not set") - -FERTISCAN_SCHEMA = os.environ.get("FERTISCAN_SCHEMA") -if FERTISCAN_SCHEMA is None or FERTISCAN_SCHEMA == "": - # raise ValueError("FERTISCAN_SCHEMA is not set") - print("Warning: FERTISCAN_SCHEMA not set") -# def connect_db(): -# """Connect to the postgresql database and return the connection.""" -# connection = psycopg.connect( -# conninfo=NACHET_DB_URL, -# autocommit=False, -# options=f"-c search_path={NACHET_SCHEMA},public") -# assert connection.info.encoding == 'utf-8', ( -# 'Encoding is not UTF8: ' + connection.info.encoding) -# # psycopg.extras.register_uuid() -# return connection - - -def connect_db(conn_str : str = NACHET_DB_URL, schema :str = NACHET_SCHEMA): +def connect_db(conn_str: str, schema: str): """Connect to the postgresql database and return the connection.""" connection = psycopg.connect( conninfo=conn_str, @@ -63,14 +34,15 @@ def end_query(connection, cursor): connection.close() -def create_search_path(connection, cur,schema = NACHET_SCHEMA): +def create_search_path(connection, cur, schema): cur.execute(f"""SET search_path TO "{schema}";""") connection.commit() + if __name__ == "__main__": - connection = connect_db(FERTISCAN_DB_URL,FERTISCAN_SCHEMA) - print(FERTISCAN_DB_URL) + connection = connect_db("test", "test") + print("test") cur = cursor(connection) - create_search_path(connection, cur,FERTISCAN_SCHEMA) + create_search_path(connection, cur, "test") end_query(connection, cur) print("Connection to the database successful") diff --git a/datastore/db/bytebase/schema_nachet_0.0.11.sql b/datastore/db/bytebase/schema_nachet_0.0.11.sql index 63b54df9..c7320db5 100644 --- a/datastore/db/bytebase/schema_nachet_0.0.11.sql +++ b/datastore/db/bytebase/schema_nachet_0.0.11.sql @@ -56,6 +56,7 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'nache "inference" json NOT NULL, "picture_id" uuid NOT NULL, "upload_date" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + "update_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, "user_id" uuid NOT NULL REFERENCES "nachet_0.0.11".users(id), "feedback_user_id" uuid, "verified" boolean DEFAULT false NOT NULL, @@ -165,6 +166,41 @@ FOR EACH ROW WHEN (NEW.verified = true) EXECUTE FUNCTION verified_inference(); + + -- Trigger function for the `inference` table + CREATE OR REPLACE FUNCTION "nachet_0.0.11".update_inference_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.update_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + -- Trigger for the `inference` table + CREATE TRIGGER inference_update_before + BEFORE UPDATE ON "nachet_0.0.11".inference + FOR EACH ROW + EXECUTE FUNCTION "nachet_0.0.11".update_inference_timestamp(); + + -- Trigger function for the `object` table + CREATE OR REPLACE FUNCTION "nachet_0.0.11".update_object_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.update_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + updated_at + + -- Trigger for the `inference` table + CREATE TRIGGER object_update_before + BEFORE UPDATE ON "nachet_0.0.11".object + FOR EACH ROW + EXECUTE FUNCTION "nachet_0.0.11".update_object_timestamp(); + + + INSERT INTO "nachet_0.0.11".seed(name) VALUES ('Brassica napus'), ('Brassica juncea'), diff --git a/datastore/db/queries/inference/__init__.py b/datastore/db/queries/inference/__init__.py index 8d3ae70f..5a84f78e 100644 --- a/datastore/db/queries/inference/__init__.py +++ b/datastore/db/queries/inference/__init__.py @@ -107,7 +107,8 @@ def set_inference_feedback_user_id(cursor, inference_id, user_id): id = %s """ cursor.execute(query, (user_id,inference_id)) - except Exception: + except Exception as e: + print(e) raise Exception(f"Error: could not set feedback_user_id {user_id} for inference {inference_id}") @@ -274,7 +275,7 @@ def get_inference_object(cursor, inference_object_id: str): valid, top_id, upload_date, - updated_at + update_at FROM object WHERE @@ -330,7 +331,8 @@ def set_inference_object_top_id(cursor, inference_object_id: str, top_id:str): UPDATE object SET - top_id = %s + top_id = %s, + update_at = now() WHERE id = %s """ @@ -379,8 +381,7 @@ def set_inference_object_verified_id(cursor, inference_object_id: str, verified_ UPDATE object SET - verified_id = %s, - updated_at = CURRENT_TIMESTAMP + verified_id = %s WHERE id = %s """ diff --git a/datastore/db/queries/user/__init__.py b/datastore/db/queries/user/__init__.py index c0044c40..186e5b72 100644 --- a/datastore/db/queries/user/__init__.py +++ b/datastore/db/queries/user/__init__.py @@ -247,7 +247,7 @@ def get_default_picture_set(cursor, user_id: str): - user_id (str): The UUID of the user Returns: - - The default picture set of the user. + - The default picture set id of the user. """ try: if not is_a_user_id(cursor=cursor, user_id=user_id): diff --git a/doc/fertiScan-architecture.md b/doc/FertiScan/fertiScan-architecture.md similarity index 100% rename from doc/fertiScan-architecture.md rename to doc/FertiScan/fertiScan-architecture.md diff --git a/doc/deployment-mass-import.md b/doc/Nachet/deployment-mass-import.md similarity index 100% rename from doc/deployment-mass-import.md rename to doc/Nachet/deployment-mass-import.md diff --git a/doc/inference-feedback.md b/doc/Nachet/inference-feedback.md similarity index 100% rename from doc/inference-feedback.md rename to doc/Nachet/inference-feedback.md diff --git a/doc/inference-results.md b/doc/Nachet/inference-results.md similarity index 100% rename from doc/inference-results.md rename to doc/Nachet/inference-results.md diff --git a/doc/nachet-architecture.md b/doc/Nachet/nachet-architecture.md similarity index 100% rename from doc/nachet-architecture.md rename to doc/Nachet/nachet-architecture.md diff --git a/doc/nachet-manage-folders.md b/doc/Nachet/nachet-manage-folders.md similarity index 100% rename from doc/nachet-manage-folders.md rename to doc/Nachet/nachet-manage-folders.md diff --git a/doc/trusted-user-upload.md b/doc/Nachet/trusted-user-upload.md similarity index 100% rename from doc/trusted-user-upload.md rename to doc/Nachet/trusted-user-upload.md diff --git a/doc/bytebase-doc.md b/doc/bytebase-doc.md index e1b436ee..4242878c 100644 --- a/doc/bytebase-doc.md +++ b/doc/bytebase-doc.md @@ -1,11 +1,10 @@ -# Workflow +# Bytebase -``` mermaid ---- -title: Bytebase workflow ---- -flowchart LR; +## Context - Project --> Database - Project --> GitOps -``` +We are storing our database architecture in a folder named bytebase. +[Bytebase](https://www.bytebase.com/docs/introduction/what-is-bytebase/) is a +version management tool for database, just like github. It also enables us to +connect to a database server on a cloud service. This is important because the +network policy we are working under doesn't allow us to make connection on port +1532. diff --git a/doc/datastore.md b/doc/datastore.md index 534e3881..eede97f9 100644 --- a/doc/datastore.md +++ b/doc/datastore.md @@ -46,7 +46,7 @@ your valuable data. ``` mermaid --- -title: Nachet Layers +title: Project Layers --- flowchart LR; FE(Frontend) @@ -68,8 +68,12 @@ flowchart LR; ``` ## Database Architecture + + For more detail on each app database architecture go check [Nachet + Architecture](Nachet/nachet-architecture.md) and [Fertiscan + Architecture](FertiScan/fertiScan-architecture.md). -### Needs +### Global Needs - A User must be able to take a picture on the app and it must be saved in the blob Storage. @@ -78,6 +82,3 @@ flowchart LR; - A User can verify the result of a picture that went through the pipeline and the changes are saved for training. - -- A User can verify an inference by confirming the positive result, or entering - the right value. diff --git a/doc/metadata-doc.md b/doc/metadata-doc.md index 73d921a9..aa7ec342 100644 --- a/doc/metadata-doc.md +++ b/doc/metadata-doc.md @@ -2,118 +2,39 @@ ## Context +The Datastore receives from its Backend JSON files related to pictures which we +also have to register. Therefore, we would need a module in place to manipulate +all the JSON and format them for our DB needs. The modules needs to also be able +to rebuild a JSON to be used by the other layers (BE & FE) + The following documentation provide an overview of the metadata importation -process for the Nachet pipeline. We outline each steps of the workflow, -illustrating how data progresses until it becomes usable by our models. +process . We outline each steps of the workflow, illustrating how data +progresses until it becomes usable by our applications. + Additionally, the upcoming process are showcased with the expected files structure. ``` mermaid --- -title: Nachet data upload +title: Data upload --- flowchart LR; FE(Frontend) BE(Backend) - file[/Folder/] classDef foo stroke:#f00 - file --> FE FE-->BE subgraph dbProcess [New Process] MD(Datastore) DB[(Database)] blob[(Blob)] end - BE -- TODO --- MD + BE -- Work in progress --- MD MD --> DB MD --> blob - -``` - -As shown above, we are currently working on a process to validate the files -uploaded to the cloud. However, since Nachet is still a work in progress, here's -the current workflow for our user to upload their images for the models. - -## Workflow: Metadata upload to Azure cloud (Currently) - -``` mermaid - -sequenceDiagram; - actor User - actor DataScientist - participant Azure Portal - participant Azure Storage Explorer - - alt New User - User->>DataScientist: Storage subscription key request - Note right of User: Email - DataScientist->>Azure Portal: Create Storage Blob - create participant Azure Storage - Azure Portal-)Azure Storage: Create() - DataScientist->>Azure Storage: Retrieve Key - DataScientist->>User: Send back subscription key - Note left of DataScientist: Email - User-)Azure Storage Explorer: SubscribeToStorage(key) - end - loop for each project Folder - User-)Azure Storage Explorer: Upload(Folder) - Azure Storage Explorer-) Azure Storage: Save(folder) - end - -``` - -This workflow showcase the two options that a user will face to upload data. The -first one being he's a first time user. Therefore, the current process for a -first time user is to contact the AI-Lab team and subscribe to the Blob storage -with a given subscription key. - -## Sequence of processing metadata for model (Currently) - -``` mermaid - -sequenceDiagram; - actor DataScientist - participant Azure Portal - participant Notebook - participant Azure Storage - DataScientist->>Azure Storage: Check files structure - alt Wrong structure - DataScientist->>Azure Portal: Create Alt. Azure Storage - Azure Portal-)Azure Storage: Create(Alt) - create participant Azure Storage Alt - DataScientist-)Azure Storage Alt: Copy files - DataScientist-)Azure Storage Alt: Rework structure + Edit files/folders - DataScientist-)Azure Storage Alt: Replace Storage content with Alt. Storage content - destroy Azure Storage Alt - Azure Storage Alt -) Azure Storage: Export files - end - DataScientist->>Azure Storage: Retrieve extraction code - DataScientist->>Notebook: Edit parameters of extraction code commands - - DataScientist-) Notebook: Run extraction code - Note left of Notebook: output source needs to be specified - Notebook -) Azure Storage: Processing files into metadata - DataScientist->> Azure Storage: Use files to train the model - ``` -This sequence illustrate the manual task done by our team to maintain the -storage of user's data. - -### Legend - -| Element | Description | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------- | -| User | Anyone wanting to upload data. | -| DataScientist | Member of the AI-Lab Team. | -| Azure Portal | Interface managing Azure's services | -| Azure Storage | Interface storing data in the cloud. | -| Azure Storage Explorer | Application with GUI offering a user friendly access to a Azure Storage without granting full acess To the Azure Services. | -| NoteBook | Azure Service enabling to run code with Azure Storage structure | -| Folder | All project folder should follow the files structure presented bellow. | - ## Development As explained in the context we aim to implement an architecture for user @@ -123,8 +44,8 @@ will be able to remove all manual metadata maintenance. Once the validation process is complete, the upload process will come into play, enabling the distribution of the files between the BLOB storage and a PostgreSQL database. - We are currently working on such process, which will be handled by the backend - part of nachet once it is finished. + We are currently improving the process to make it more efficient across all our + project. ``` mermaid @@ -152,177 +73,6 @@ flowchart LR; ``` -## Finished processes - -### [Deployment mass import](deployment-mass-import.md) - -### [Trusted User Upload](trusted-user-upload.md) - -## Database - -We plan on storing the metadata of the user's files in a postgreSQL Database. -The database should have the following structure: - -``` mermaid - ---- -title: Nachet DB Structure ---- -erDiagram - user{ - uuid id PK - string email - timestamp registration_date - timestamp updated_at - integer permission_id - } - picture_set{ - uuid id PK - json picture_set - uuid owner_id FK - timestamp upload_date - } - picture{ - uuid id PK - json picture - uuid picture_set_id FK - uuid parent FK - int nb_object - boolean verified - timestamp upload_date - } - picture_seed{ - uuid id PK - uuid picture_id FK - uuid seed_id FK - timestamp upload_date - } - seed{ - uuid id PK - string name - json information - uuid object_type_id - timestamp upload_date - } - object_type{ - integer id PK - text name - } - inference{ - uuid id PK - json inference - uuid picture_id FK - uuid user_id FK - timestamp upload_date - } - model{ - uuid id PK - text name - text endpoint_name - integer task_id FK - uuid active_version FK - } - model_version{ - uuid id PK - uuid model_id FK - json data - text version - timestamp upload_date - } - object{ - uuid id PK - json box_metadata - uuid inference_id FK - integer type_id - uuid verified_id - boolean valid - uuid top_inference FK - timestamp upload_date - timestamp updated_at - } - seed_object{ - uuid id PK - uuid seed_id FK - uuid object_id FK - } - pipeline{ - uuid id PK - text name - boolean active - boolean is_default - json data - } - pipeline_model{ - uuid id PK - uuid pipeline_id FK - uuid model_id FK - } - pipeline_default{ - uuid id PK - uuid pipeline_id FK - uuid user_id FK - timestamp update_date - } - group{ - uuid id PK - text name - int permission_id FK - uuid owner_id FK - timestamp upload_date - } - permission{ - int id - text name - } - user_group{ - uuid id - uuid user_id - uuid group_id - timestamp upload_date - - } - group_container{ - uuid id - uuid group_id - uuid container_id - timestamp upload_date - } - container{ - uuid id PK - uuid owner_id FK - boolean public - timestamp creation_date - timestamp updated_at - } - - user ||--|{ picture_set: uploads - picture_set ||--o{picture: contains - picture ||--o{picture: cropped - picture |o--o{picture_seed: has - picture_seed }o--o| seed: has - object_type ||--|| seed: is - user ||--o{ inference: requests - inference ||--|| picture: infers - inference }o--|| pipeline: uses - inference ||--o{ object: detects - object ||--o{ seed_object: is - seed_object }o--|| seed: is - object }o--|| object_type: is - user ||--||pipeline_default: select - pipeline_default }o--||pipeline: is - pipeline }o--o{ pipeline_model: uses - model }o--o{ pipeline_model: uses - model_version ||--o{ model: has - user }o--o{ user_group: apart - user_group }o--o{group: represent - permission ||--|| user: has - permission ||--|| group: has - group }o--o{group_container: has - container }o--o{group_container: represent - container ||--o{picture_set: contains - -``` - ## Blob Storage Finally the picture uploaded by the users will need to be stored in a blob @@ -356,8 +106,8 @@ Storage account: ## Consequences - Implementing this structure and introducing the new backend features in Nachet - will result in the following impact: + Implementing this structure and introducing the new backend features will + result in the following impact: - **Automation of file structure maintenance:** This process will autonomously diff --git a/pyproject.toml b/pyproject.toml index d67925b3..1869ab5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,10 @@ build-backend = "setuptools.build_meta" [project] name = "nachet_datastore" -version = "0.0.1" +version = "0.1.0" authors = [ { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, + { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca"} ] description = "Data management python layer" readme = "README.md" @@ -24,10 +25,10 @@ license = { file = "LICENSE" } keywords = ["Nachet","ailab"] [tool.setuptools] -packages = ["datastore", "datastore.db", "datastore.bin", +packages = ["datastore", "datastore.Nachet", "datastore.db", "datastore.bin", "datastore.db.queries", "datastore.db.queries.picture", "datastore.db.queries.seed", "datastore.db.queries.user", -"datastore.db.queries.machine_learning","datastore.db.queries.inference", "datastore.db.queries.analysis", +"datastore.db.queries.machine_learning","datastore.db.queries.inference", "datastore.db.metadata","datastore.db.metadata.picture", "datastore.db.metadata.picture_set", "datastore.db.metadata.validator", "datastore.db.metadata.machine_learning","datastore.db.metadata.inference", diff --git a/pyproject_Nachet.toml b/pyproject_Nachet.toml new file mode 100644 index 00000000..1869ab5a --- /dev/null +++ b/pyproject_Nachet.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "nachet_datastore" +version = "0.1.0" +authors = [ + { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" }, + { name="Sylvanie You", email="Sylvanie.You@inspection.gc.ca"} +] +description = "Data management python layer" +readme = "README.md" +requires-python = ">=3.11" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dynamic = ["dependencies"] + + +license = { file = "LICENSE" } + +keywords = ["Nachet","ailab"] + +[tool.setuptools] +packages = ["datastore", "datastore.Nachet", "datastore.db", "datastore.bin", +"datastore.db.queries", "datastore.db.queries.picture", +"datastore.db.queries.seed", "datastore.db.queries.user", +"datastore.db.queries.machine_learning","datastore.db.queries.inference", +"datastore.db.metadata","datastore.db.metadata.picture", "datastore.db.metadata.picture_set", +"datastore.db.metadata.validator", +"datastore.db.metadata.machine_learning","datastore.db.metadata.inference", +"datastore.blob","datastore.blob.azure_storage_api"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html +# [tool.setuptools.packages.find] +# where = ["ailab"] + +[project.urls] +"Homepage" = "https://github.com/ai-cfia/nachet-datastore" +"Bug Tracker" = "https://github.com/ai-cfia/nachet-datastore/issues" +Repository = "https://github.com/ai-cfia/nachet-datastore" diff --git a/pyproject_fertiscan.toml b/pyproject_fertiscan.toml new file mode 100644 index 00000000..41b4a5f2 --- /dev/null +++ b/pyproject_fertiscan.toml @@ -0,0 +1,47 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "fertiscan_datastore" +version = "0.0.1" +authors = [ + { name="Francois Werbrouck", email="francois.werbrouck@inspection.gc.ca" } +] +description = "Data management python layer" +readme = "README.md" +requires-python = ">=3.11" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dynamic = ["dependencies"] + + +license = { file = "LICENSE" } + +keywords = ["FertiScan","ailab"] + +[tool.setuptools] +packages = ["datastore", "datastore.FertiScan", "datastore.bin","datastore.db", +"datastore.db.queries", "datastore.db.queries.picture", +"datastore.db.queries.user", +"datastore.db.queries.inspection","datastore.db.queries.label","datastore.db.queries.metric", +"datastore.db.queries.nutrients","datastore.db.queries.organization","datastore.db.queries.specification", +"datastore.db.queries.sub_label", +"datastore.db.metadata","datastore.db.metadata.picture", "datastore.db.metadata.picture_set", +"datastore.db.metadata.validator", +"datastore.blob","datastore.blob.azure_storage_api"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html +# [tool.setuptools.packages.find] +# where = ["ailab"] + +[project.urls] +"Homepage" = "https://github.com/ai-cfia/nachet-datastore" +"Bug Tracker" = "https://github.com/ai-cfia/nachet-datastore/issues" +Repository = "https://github.com/ai-cfia/nachet-datastore" diff --git a/tests/Nachet/__init__.py b/tests/Nachet/__init__.py new file mode 100644 index 00000000..fba0404a --- /dev/null +++ b/tests/Nachet/__init__.py @@ -0,0 +1,4 @@ +import unittest + +if __name__ == "__main__": + unittest.main() diff --git a/tests/Nachet/db/test_inference.py b/tests/Nachet/db/test_inference.py new file mode 100644 index 00000000..48808710 --- /dev/null +++ b/tests/Nachet/db/test_inference.py @@ -0,0 +1,447 @@ +""" +This is a test script for the database packages. +It tests the functions in the user, seed and picture modules. +""" + +import unittest +import uuid +import json +import os +from PIL import Image +import io +import base64 +from time import sleep +from unittest.mock import MagicMock + +import datastore.db.__init__ as db +from datastore.db.metadata import picture as picture_data +from datastore.db.metadata import picture_set as picture_set_data +from datastore.db.metadata import validator +from datastore.db.queries import inference, picture, seed, user + +DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") +if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": + raise ValueError("NACHET_DB_URL is not set") + +DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") +if DB_SCHEMA is None or DB_SCHEMA == "": + raise ValueError("NACHET_SCHEMA_TESTING is not set") + + +# -------------------- INFERENCE FUNCTIONS -------------------- +class test_inference_functions(unittest.TestCase): + def setUp(self): + # prepare the connection and cursor + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = db.cursor(self.con) + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + + # prepare the seed + self.seed_name = "test seed" + self.seed_id = seed.new_seed(self.cursor, self.seed_name) + + # prepare the user + self.user_id = user.register_user(self.cursor, "test@email") + + # prepare the picture_set and picture + self.image = Image.new("RGB", (1980, 1080), "blue") + self.image_byte_array = io.BytesIO() + self.image.save(self.image_byte_array, format="TIFF") + self.pic_encoded = base64.b64encode(self.image_byte_array.getvalue()).decode( + "utf8" + ) + self.picture_set = picture_set_data.build_picture_set(self.user_id, 1) + self.nb_seed = 1 + self.picture = picture_data.build_picture( + self.pic_encoded, "www.link.com", self.nb_seed, 1.0, "" + ) + self.picture_set_id = picture.new_picture_set( + self.cursor, self.picture_set, self.user_id + ) + self.picture_id = picture.new_picture( + self.cursor, self.picture, self.picture_set_id, self.seed_id, self.nb_seed + ) + with open("tests/Nachet/inference_example.json", "r") as f: + self.inference = json.loads(f.read()) + self.inference_trim = ( + '{"filename": "inference_example", "totalBoxes": 1, "totalBoxes": 1}' + ) + self.type = 1 + + def tearDown(self): + self.con.rollback() + db.end_query(self.con, self.cursor) + + def test_new_inference(self): + """ + This test checks if the new_inference function returns a valid UUID + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + + self.assertTrue( + validator.is_valid_uuid(inference_id), + "The inference_id is not a valid UUID", + ) + + def test_new_inference_error(self): + """ + This test checks if the new_inference function raises an exception when the connection fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(inference.InferenceCreationError): + inference.new_inference( + mock_cursor, + self.inference_trim, + self.user_id, + self.picture_id, + self.type, + ) + + def test_new_inference_obj(self): + """ + This test checks if the new_inference_object function returns a valid UUID + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + for box in self.inference["boxes"]: + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(box), self.type + ) + self.assertTrue( + validator.is_valid_uuid(inference_obj_id), + "The inference_obj_id is not a valid UUID", + ) + + def test_new_inference_obj_error(self): + """ + This test checks if the new_inference_object function raises an exception when the connection fails + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + + with self.assertRaises(inference.InferenceCreationError): + inference.new_inference_object( + mock_cursor, inference_id, self.inference["boxes"][0], self.type + ) + + def test_new_seed_object(self): + """ + This test checks if the new_seed_object function returns a valid UUID + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + for box in self.inference["boxes"]: + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(box), self.type + ) + seed_obj_id = inference.new_seed_object( + self.cursor, self.seed_id, inference_obj_id, box["score"] + ) + self.assertTrue( + validator.is_valid_uuid(seed_obj_id), + "The seed_obj_id is not a valid UUID", + ) + + def test_new_seed_object_error(self): + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(inference.SeedObjectCreationError): + inference.new_seed_object(mock_cursor, self.seed_id, inference_obj_id, 32.1) + + def test_get_inference(self): + """ + This test checks if the get_inference function returns a correctly build inference + TODO : Add test for not existing inference + """ + inference_trim = json.loads(self.inference_trim) + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_data = inference.get_inference(self.cursor, str(inference_id)) + # inference_json=json.loads(inference_data) + self.assertGreaterEqual( + len(inference_trim), + len(inference_data), + "The inference is not correctly build and has more keys than expected", + ) + for key in inference_trim: + self.assertTrue( + key in inference_data, f"The key: {key} is not in the inference" + ) + self.assertEqual( + inference_trim[key], + inference_data[key], + f"The value ({inference_data[key]}) of the key: {key} is not the same as the expected one: {inference_trim[key]}", + ) + + def test_get_inference_object(self): + """ + This test checks if the get_inference_object function returns a correctly build object + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + + inference_obj = inference.get_inference_object( + self.cursor, str(inference_obj_id) + ) + + self.assertEqual( + len(inference_obj), + 10, + "The inference object hasn't the number of keys expected", + ) + self.assertEqual( + inference_obj[0], + inference_obj_id, + "The inference object id is not the same as the expected one", + ) + + def test_get_inference_object_error(self): + """ + This test checks if the get_inference_object function raise an error if the inference oject does not exist + """ + inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = "00000000-0000-0000-0000-000000000000" + + with self.assertRaises(Exception): + inference.get_inference_object(self.cursor, str(inference_obj_id)) + + def test_get_objects_by_inference(self): + """ + This test checks if the get_objects_by_inference function returns the corrects objects for an inference + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + total_boxes = len(self.inference["boxes"]) + objects_id = [] + for box in self.inference["boxes"]: + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(box), self.type + ) + objects_id.append(inference_obj_id) + + objects = inference.get_objects_by_inference(self.cursor, inference_id) + self.assertEqual( + len(objects), + total_boxes, + "The number of objects is not the same as the expected one", + ) + for object in objects: + self.assertEqual( + object[2], + inference_id, + "The inference id is not the same as the expected one", + ) + self.assertTrue( + object[0] in objects_id, + "The object id is not in the list of expected objects", + ) + + def test_get_inference_object_top_id(self): + """ + This test checks if the get_inference_object_top_id function returns the correct top_id of an inference object + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + seed_obj_id = inference.new_seed_object( + self.cursor, + self.seed_id, + inference_obj_id, + self.inference["boxes"][0]["score"], + ) + + inference.set_inference_object_top_id( + self.cursor, inference_obj_id, seed_obj_id + ) + top_id = inference.get_inference_object_top_id(self.cursor, inference_obj_id) + + self.assertEqual( + seed_obj_id, top_id, "The verified_id is not the same as the expected one" + ) + + def test_set_inference_object_verified_id(self): + """ + This test checks if the set_inference_object_verified_id function returns a correctly update inference object + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + inference.get_inference_object( + self.cursor, inference_obj_id + ) + # print(previous_inference_obj) + seed_obj_id = inference.new_seed_object( + self.cursor, + self.seed_id, + inference_obj_id, + self.inference["boxes"][0]["score"], + ) + # Sleep to see a difference in the updated_at date of the object + sleep(3) + + inference.set_inference_object_verified_id( + self.cursor, inference_obj_id, seed_obj_id + ) + inference_obj = inference.get_inference_object(self.cursor, inference_obj_id) + # print(inference_obj) + self.assertEqual( + str(inference_obj[4]), + str(seed_obj_id), + "The verified_id is not the same as the expected one", + ) + + def test_set_inference_object_valid(self): + """ + This test checks if the set_inference_object_verified_id function returns a correctly update inference object + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + inference.get_inference_object( + self.cursor, inference_obj_id + ) + # Sleep to see a difference in the updated_at date of the object + sleep(3) + + inference.set_inference_object_valid(self.cursor, inference_obj_id, True) + inference_obj = inference.get_inference_object(self.cursor, inference_obj_id) + self.assertTrue( + str(inference_obj[5]), + "The object validity is not the same as the expected one", + ) + + inference.set_inference_object_valid(self.cursor, inference_obj_id, False) + inference_obj = inference.get_inference_object(self.cursor, inference_obj_id) + self.assertEqual( + str(inference_obj[6]), + "False", + "The object validity is not the same as the expected one", + ) + + def test_is_inference_verified(self): + """ + Test if is_inference_verified function correctly returns the inference status + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + + verified = inference.is_inference_verified(self.cursor, inference_id) + self.assertFalse(verified, "The inference verified field should be False") + + inference.set_inference_verified(self.cursor, inference_id, True) + + verified = inference.is_inference_verified(self.cursor, inference_id) + self.assertTrue(verified, "The inference should be fully verified") + + def test_verify_inference_status(self): + """ + Test if verify_inference_status function correctly updates the inference status + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + seed_obj_id = inference.new_seed_object( + self.cursor, + self.seed_id, + inference_obj_id, + self.inference["boxes"][0]["score"], + ) + + inference.verify_inference_status(self.cursor, inference_id, self.user_id) + + verified = inference.is_inference_verified(self.cursor, inference_id) + self.assertFalse(verified, "The inference verified field should be False") + + inference.set_inference_object_valid(self.cursor, inference_obj_id, True) + inference.set_inference_object_verified_id( + self.cursor, inference_obj_id, seed_obj_id + ) + + inference.verify_inference_status(self.cursor, inference_id, self.user_id) + verified = inference.is_inference_verified(self.cursor, inference_id) + self.assertTrue(verified, "The inference should be fully verified") + + def test_get_seed_object_id(self): + """ + Test if get_seed_object_id function correctly returns the seed object id + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + seed_obj_id = inference.new_seed_object( + self.cursor, + self.seed_id, + inference_obj_id, + self.inference["boxes"][0]["score"], + ) + + fetched_seed_obj_id = inference.get_seed_object_id( + self.cursor, self.seed_id, inference_obj_id + ) + self.assertEqual( + seed_obj_id, + fetched_seed_obj_id, + "The fetched seed object id is not the same as the expected one", + ) + + def test_get_not_seed_object_id(self): + """ + Test if get_seed_object_id function correctly returns the seed object id + """ + inference_id = inference.new_inference( + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + ) + inference_obj_id = inference.new_inference_object( + self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + ) + inference.new_seed_object( + self.cursor, + self.seed_id, + inference_obj_id, + self.inference["boxes"][0]["score"], + ) + mock_seed_id = str(uuid.uuid4()) + fetched_seed_obj_id = inference.get_seed_object_id( + self.cursor, mock_seed_id, inference_obj_id + ) + self.assertTrue( + fetched_seed_obj_id is None, "The fetched seed object id should be None" + ) diff --git a/tests/Nachet/db/test_metadata.py b/tests/Nachet/db/test_metadata.py new file mode 100644 index 00000000..9bf57fa1 --- /dev/null +++ b/tests/Nachet/db/test_metadata.py @@ -0,0 +1,224 @@ +import datastore.db.metadata.inference as inference +import datastore.db.metadata.machine_learning as ml_data +import json +import unittest + +PIC_LINK = "test.com" + +PIC_PATH = "img/test_image.tiff" + + +class test_inference_functions(unittest.TestCase): + def setUp(self): + self.filename = "test.jpg" + self.overlapping = False + self.overlapping_indices = 0 + self.total_boxes = 1 + with open("tests/Nachet/inference_example.json") as f: + self.inference_exemple = json.load(f) + self.filename = self.inference_exemple["filename"] + self.labelOccurrence = self.inference_exemple["labelOccurrence"] + self.total_boxes = self.inference_exemple["totalBoxes"] + self.boxes = self.inference_exemple["boxes"] + + def test_build_inference_import(self): + """ + This test checks if the build_inference_import function returns a valid JSON object + """ + mock_inference = { + "filename": self.filename, + "labelOccurrence": self.labelOccurrence, + "totalBoxes": self.total_boxes, + } + inference_tested = inference.build_inference_import(self.inference_exemple) + inference_tested = json.loads(inference_tested) + self.assertGreaterEqual( + len(mock_inference), + len(inference_tested), + "The inference returned has too many keys", + ) + for key, value in mock_inference.items(): + self.assertTrue( + key in inference_tested, f"{key} should be in the inference object" + ) + self.assertEqual( + inference_tested[key], + value, + f"{key} should be {value}", + ) + + def test_build_object_import(self): + """ + This test checks if the build_object_import function returns a valid JSON object + """ + object = self.boxes[0] + object_tested = inference.build_object_import(object) + data = json.loads(object_tested) + mock_object = { + "box": {"topX": 0.0, "topY": 0.0, "bottomX": 0.0, "bottomY": 0.0}, + "color": "#ff0", + } + for key, value in mock_object.items(): + self.assertEqual( + data[key], + value, + f"{key} should be {value}", + ) + + +class test_machine_learning_functions(unittest.TestCase): + def setUp(self): + with open("tests/Nachet/ml_structure_exemple.json", "r") as file: + self.ml_structure = json.load(file) + self.model_name = "that_model_name" + self.model_id = "48efe646-5210-4761-908e-a06f95f0c344" + self.model_endpoint = "https://that-model.inference.ml.com/score" + self.task = "classification" + self.version = 5 + self.data = { + "creation_date": "2024-01-01", + "created_by": "Avery GoodDataScientist", + "Accuracy": 0.6908, + } + self.mock_model = { + "task": "classification", + "endpoint": "https://that-model.inference.ml.com/score", + "api_key": "SeCRetKeys", + "content_type": "application/json", + "deployment_platform": "azure", + "endpoint_name": "that-model-endpoint", + "model_name": "that_model_name", + "model_id": "48efe646-5210-4761-908e-a06f95f0c344", + "created_by": "Avery GoodDataScientist", + "creation_date": "2024-01-01", + "version": 5, + "description": "Model Description", + "job_name": "Job Name", + "dataset": "Dataset Description", + "Accuracy": 0.6908, + } + self.pipeline_name = "First Pipeline" + self.pipeline_id = "48efe646-5210-4761-908e-a06f95f0c344" + self.pipeline_default = True + self.model_ids = ["that_model_name", "other_model_name"] + self.mock_pipeline = { + "models": ["that_model_name", "other_model_name"], + "pipeline_id": "48efe646-5210-4761-908e-a06f95f0c344", + "pipeline_name": "First Pipeline", + "model_name": "First Pipeline", + "created_by": "Avery GoodDataScientist", + "creation_date": "2024-01-01", + "version": 1, + "description": "Pipeline Description", + "job_name": "Job Name", + "dataset": "Dataset Description", + "Accuracy": 0.6908, + "default": True, + } + + def test_build_model_import(self): + """ + This test checks if the build_model_import function returns a valid JSON object + """ + model = self.ml_structure["models"][0] + model_import = ml_data.build_model_import(model) + model_data = json.loads(model_import) + self.assertLessEqual( + len(model_data), + len(self.mock_model), + "The returned model data should have more key than expected", + ) + for key in model_data: + self.assertEqual( + self.mock_model[key], + model_data[key], + f"{key} should be {self.mock_model[key]}", + ) + + def test_missing_key_build_model_import(self): + """ + This test checks if the build_model_import function raises an error when a key is missing + """ + model = self.ml_structure["models"][0] + model.pop("api_key") + self.assertRaises(ml_data.MissingKeyError, ml_data.build_model_import, model) + + def test_build_model_export(self): + """ + This test checks if the build_model_export function returns a valid JSON object + """ + model_export = ml_data.build_model_export( + self.data, + self.model_id, + self.model_name, + self.model_endpoint, + self.task, + self.version, + ) + self.assertLessEqual( + len(model_export), + len(self.mock_model), + "The returned model data should have more key than expected", + ) + for key in model_export: + self.assertTrue(key in self.mock_model, f"The key:{key} was not expected") + self.assertEqual( + self.mock_model[key], + model_export[key], + f"{key} should be {self.mock_model[key]}", + ) + + def test_build_pipeline_import(self): + """ + This test checks if the build_pipeline_import function returns a valid JSON object + """ + pipeline = self.ml_structure["pipelines"][0] + pipeline_import = ml_data.build_pipeline_import(pipeline) + pipeline_data = json.loads(pipeline_import) + self.assertLessEqual( + len(pipeline_data), + len(self.mock_pipeline), + "The returned pipeline data should have more key than expected", + ) + for key in pipeline_data: + self.assertEqual( + pipeline_data[key], + self.mock_pipeline[key], + f"The pipeline data for the key:{key} should be {self.mock_pipeline[key]}", + ) + + def test_missing_key_build_pipeline_import(self): + """ + This test checks if the build_pipeline_import function raises an error when a key is missing + """ + pipeline = self.ml_structure["pipelines"][0] + pipeline.pop("models") + self.assertRaises( + ml_data.MissingKeyError, ml_data.build_pipeline_import, pipeline + ) + + def test_build_pipeline_export(self): + """ + This test checks if the build_pipeline_export function returns a valid JSON object + """ + pipeline_export = ml_data.build_pipeline_export( + self.data, + self.pipeline_name, + self.pipeline_id, + self.pipeline_default, + self.model_ids, + ) + self.assertLessEqual( + len(pipeline_export), + len(self.mock_pipeline), + "The returned pipeline data should have more key than expected", + ) + for key in pipeline_export: + self.assertTrue( + key in self.mock_pipeline, f"The key:{key} was not expected" + ) + self.assertEqual( + self.mock_pipeline[key], + pipeline_export[key], + f"{key} should be {self.mock_pipeline[key]}", + ) diff --git a/tests/Nachet/db/test_picture.py b/tests/Nachet/db/test_picture.py new file mode 100644 index 00000000..6a2f695e --- /dev/null +++ b/tests/Nachet/db/test_picture.py @@ -0,0 +1,89 @@ +""" +This is a test script for the database packages. +It tests the functions in the user, seed and picture modules. +""" + +import unittest +import uuid +import os +from PIL import Image +import io +import base64 +from unittest.mock import MagicMock + +import datastore.db.__init__ as db +from datastore.db.metadata import picture as picture_data +from datastore.db.metadata import picture_set as picture_set_data +from datastore.db.metadata import validator +from datastore.db.queries import picture, seed, user + +DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") +if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": + raise ValueError("NACHET_DB_URL_TESTING is not set") + +DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") +if DB_SCHEMA is None or DB_SCHEMA == "": + raise ValueError("NACHET_SCHEMA_TESTING is not set") + + +# -------------------- PICTURE FUNCTIONS -------------------- +class test_pictures_functions(unittest.TestCase): + def setUp(self): + # prepare the connection and cursor + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = db.cursor(self.con) + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + + # prepare the seed + self.seed_name = "test seed" + self.seed_id = seed.new_seed(self.cursor, self.seed_name) + + # prepare the user + self.user_id = user.register_user(self.cursor, "test@email") + + # prepare the picture_set and picture + self.image = Image.new("RGB", (1980, 1080), "blue") + self.image_byte_array = io.BytesIO() + self.image.save(self.image_byte_array, format="TIFF") + self.pic_encoded = base64.b64encode(self.image_byte_array.getvalue()).decode( + "utf8" + ) + self.nb_seed = 1 + self.picture_set = picture_set_data.build_picture_set(self.user_id, 1) + self.picture = picture_data.build_picture( + self.pic_encoded, "www.link.com", self.nb_seed, 1.0, "" + ) + self.folder_name = "test_folder" + + def tearDown(self): + self.con.rollback() + db.end_query(self.con, self.cursor) + + def test_new_picture(self): + """ + This test checks if the new_picture function returns a valid UUID + """ + # prepare the picture_set + picture_set_id = picture.new_picture_set( + self.cursor, self.picture_set, self.user_id + ) + + # create the new picture in the db + picture_id = picture.new_picture( + self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + ) + + self.assertTrue( + validator.is_valid_uuid(picture_id), "The picture_id is not a valid UUID" + ) + + def test_new_picture_error(self): + """ + This test checks if the new_picture function raises an exception when the connection fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(picture.PictureUploadError): + picture.new_picture( + mock_cursor, self.picture, str(uuid.uuid4()), self.seed_id, self.nb_seed + ) diff --git a/tests/Nachet/db/test_seed.py b/tests/Nachet/db/test_seed.py new file mode 100644 index 00000000..ad55bda9 --- /dev/null +++ b/tests/Nachet/db/test_seed.py @@ -0,0 +1,123 @@ +""" +This is a test script for the database packages. +It tests the functions in the user, seed and picture modules. +""" + +import unittest +import os +from unittest.mock import MagicMock + +import datastore.db.__init__ as db +from datastore.db.metadata import validator +from datastore.db.queries import seed + +DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") +if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": + raise ValueError("NACHET_DB_URL_TESTING is not set") + +DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") +if DB_SCHEMA is None or DB_SCHEMA == "": + raise ValueError("NACHET_SCHEMA_TESTING is not set") + + +# -------------------- SEED FUNCTIONS -------------------- +class test_seed_functions(unittest.TestCase): + def setUp(self): + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = db.cursor(self.con) + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + self.seed_name = "test-name" + + def tearDown(self): + self.con.rollback() + db.end_query(self.con, self.cursor) + + def test_get_all_seeds_names(self): + """ + This test checks if the get_all_seeds_names function returns a list of seeds + with at least the one seed we added + """ + seed.new_seed(self.cursor, self.seed_name) + seeds = seed.get_all_seeds_names(self.cursor) + + self.assertNotEqual(len(seeds), 0) + self.assertIn((self.seed_name,), seeds) + + def test_get_all_seeds_names_error(self): + """ + This test checks if the get_all_seeds_names function raises an exception when the connection fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchall.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + seed.get_all_seeds_names(mock_cursor) + + def test_get_seed_id(self): + """ + This test checks if the get_seed_id function returns the correct UUID + """ + seed_uuid = seed.new_seed(self.cursor, self.seed_name) + fetch_id = seed.get_seed_id(self.cursor, self.seed_name) + + self.assertTrue(validator.is_valid_uuid(fetch_id)) + self.assertEqual(seed_uuid, fetch_id) + + def test_get_nonexistant_seed_id(self): + """ + This test checks if the get_seed_id function raises an exception when the seed does not exist + """ + with self.assertRaises(seed.SeedNotFoundError): + seed.get_seed_id(self.cursor, "nonexistant_seed") + + def test_get_seed_id_error(self): + """ + This test checks if the get_seed_id function raises an exception when the connection fails + """ + seed.new_seed(self.cursor, self.seed_name) + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + seed.get_seed_id(mock_cursor, self.seed_name) + + def test_new_seed(self): + """ + This test checks if the new_seed function returns a valid UUID + """ + seed_id = seed.new_seed(self.cursor, self.seed_name) + + self.assertTrue(validator.is_valid_uuid(seed_id)) + + def test_new_seed_error(self): + """ + This test checks if the new_seed function raises an exception when the connection fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(seed.SeedCreationError): + seed.new_seed(mock_cursor, self.seed_name) + + def test_is_seed_registered(self): + """ + This test checks if the is_seed_registered function returns the correct value + for a seed that is not yet registered and one that is. + """ + self.assertFalse( + seed.is_seed_registered(self.cursor, self.seed_name), + "The seed should not already be registered", + ) + + seed.new_seed(self.cursor, self.seed_name) + + self.assertTrue( + seed.is_seed_registered(self.cursor, self.seed_name), + "The seed should be registered", + ) + + def test_is_seed_registered_error(self): + """ + This test checks if the is_seed_registered function raises an exception when the connection fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + seed.is_seed_registered(mock_cursor, self.seed_name) diff --git a/tests/inference_example.json b/tests/Nachet/inference_example.json similarity index 100% rename from tests/inference_example.json rename to tests/Nachet/inference_example.json diff --git a/tests/inference_feedback_correction.json b/tests/Nachet/inference_feedback_correction.json similarity index 100% rename from tests/inference_feedback_correction.json rename to tests/Nachet/inference_feedback_correction.json diff --git a/tests/inference_feedback_perfect.json b/tests/Nachet/inference_feedback_perfect.json similarity index 100% rename from tests/inference_feedback_perfect.json rename to tests/Nachet/inference_feedback_perfect.json diff --git a/tests/inference_result.json b/tests/Nachet/inference_result.json similarity index 100% rename from tests/inference_result.json rename to tests/Nachet/inference_result.json diff --git a/tests/ml_structure_exemple.json b/tests/Nachet/ml_structure_exemple.json similarity index 100% rename from tests/ml_structure_exemple.json rename to tests/Nachet/ml_structure_exemple.json diff --git a/tests/Nachet/test_datastore.py b/tests/Nachet/test_datastore.py new file mode 100644 index 00000000..3be2c2c7 --- /dev/null +++ b/tests/Nachet/test_datastore.py @@ -0,0 +1,952 @@ +""" +This is a test script for the highest level of the datastore packages. +It tests the functions in the __init__.py files of the datastore packages. +""" + +import io +import os +import unittest +from unittest.mock import MagicMock +from PIL import Image +import json +import uuid +import asyncio +import datastore.db.__init__ as db +import datastore.__init__ as datastore +import datastore.Nachet.__init__ as Nachet +import datastore.db.metadata.validator as validator +import datastore.db.queries.seed as seed_query +from copy import deepcopy + + +DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") +if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": + raise ValueError("NACHET_DB_URL is not set") + +DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") +if DB_SCHEMA is None or DB_SCHEMA == "": + raise ValueError("NACHET_SCHEMA_TESTING is not set") + +BLOB_CONNECTION_STRING = os.environ["NACHET_STORAGE_URL_TESTING"] +if BLOB_CONNECTION_STRING is None or BLOB_CONNECTION_STRING == "": + raise ValueError("NACHET_STORAGE_URL_TESTING is not set") + +BLOB_ACCOUNT = os.environ["NACHET_BLOB_ACCOUNT_TESTING"] +if BLOB_ACCOUNT is None or BLOB_ACCOUNT == "": + raise ValueError("NACHET_BLOB_ACCOUNT is not set") + +BLOB_KEY = os.environ["NACHET_BLOB_KEY_TESTING"] +if BLOB_KEY is None or BLOB_KEY == "": + raise ValueError("NACHET_BLOB_KEY is not set") + +DEV_USER_EMAIL = os.environ.get("DEV_USER_EMAIL") +if DEV_USER_EMAIL is None or DEV_USER_EMAIL == "": + # raise ValueError("DEV_USER_EMAIL is not set") + print("Warning: DEV_USER_EMAIL not set") + + +class test_ml_structure(unittest.TestCase): + def setUp(self): + with open("tests/Nachet/ml_structure_exemple.json") as file: + self.ml_dict = json.load(file) + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = self.con.cursor() + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + + def tearDown(self): + self.con.rollback() + db.end_query(self.con, self.cursor) + + def test_import_ml_structure_from_json(self): + """ + Test the import function. + """ + asyncio.run( + Nachet.import_ml_structure_from_json_version(self.cursor, self.ml_dict) + ) + self.cursor.execute("SELECT id FROM model WHERE name='that_model_name'") + model_id = self.cursor.fetchone()[0] + self.assertTrue(validator.is_valid_uuid(str(model_id))) + self.cursor.execute("SELECT id FROM pipeline WHERE name='Second Pipeline'") + pipeline_id = self.cursor.fetchone()[0] + self.assertTrue(validator.is_valid_uuid(str(pipeline_id))) + self.cursor.execute( + "SELECT id FROM pipeline_model WHERE pipeline_id=%s AND model_id=%s", + ( + pipeline_id, + model_id, + ), + ) + self.assertTrue(validator.is_valid_uuid(self.cursor.fetchone()[0])) + + def test_get_ml_structure(self): + """ + Test the get function. + """ + + # asyncio.run(datastore.import_ml_structure_from_json_version(self.cursor,self.ml_dict)) + ml_structure = asyncio.run(Nachet.get_ml_structure(self.cursor)) + # self.assertDictEqual(ml_structure,self.ml_dict) + for pipeline in self.ml_dict["pipelines"]: + for key in pipeline: + if key != "Accuracy": + self.assertTrue( + (key in ml_structure["pipelines"][0].keys()), + f"Key {key} was not found and expected in the returned dictionary", + ) + for model in self.ml_dict["models"]: + for key in model: + if key != "Accuracy" and key != "endpoint_name": + # print(key) + self.assertTrue( + (key in ml_structure["models"][0].keys()), + f"Key {key} was not found and expected in the returned dictionary", + ) + + def test_get_ml_structure_eeror(self): + """ + Test the get version function. + """ + mock_cursor = MagicMock() + mock_cursor.fetchall.return_value = [] + with self.assertRaises(Nachet.MLRetrievalError): + asyncio.run(Nachet.get_ml_structure(mock_cursor)) + + +class test_picture(unittest.TestCase): + def setUp(self): + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = self.con.cursor() + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + self.connection_str = BLOB_CONNECTION_STRING + self.user_email = "test@email" + self.user_obj = asyncio.run( + datastore.new_user( + self.cursor, self.user_email, self.connection_str, "test-user" + ) + ) + self.image = Image.new("RGB", (1980, 1080), "blue") + self.image_byte_array = io.BytesIO() + self.image.save(self.image_byte_array, format="TIFF") + self.pic_encoded = self.image.tobytes() + self.pictures = [self.pic_encoded, self.pic_encoded, self.pic_encoded] + # self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) + self.container_name = "test-container" + self.user_id = datastore.User.get_id(self.user_obj) + self.container_client = asyncio.run( + datastore.get_user_container_client( + self.user_id, + BLOB_CONNECTION_STRING, + BLOB_ACCOUNT, + BLOB_KEY, + "test-user", + ) + ) + self.seed_name = "test-name" + self.seed_id = seed_query.new_seed(self.cursor, self.seed_name) + with open("tests/Nachet/inference_result.json") as file: + self.inference = json.load(file) + self.folder_name = "test_folder" + + def tearDown(self): + self.con.rollback() + self.container_client.delete_container() + db.end_query(self.con, self.cursor) + + def test_upload_picture_unknown(self): + """ + Test the upload picture function. + """ + picture_id = asyncio.run( + Nachet.upload_picture_unknown( + self.cursor, self.user_id, self.pic_encoded, self.container_client + ) + ) + self.assertTrue(validator.is_valid_uuid(picture_id)) + + def test_register_inference_result(self): + """ + Test the register inference result function. + """ + picture_id = asyncio.run( + Nachet.upload_picture_unknown( + self.cursor, self.user_id, self.pic_encoded, self.container_client + ) + ) + model_id = "test_model_id" + + result = asyncio.run( + Nachet.register_inference_result( + self.cursor, self.user_id, self.inference, picture_id, model_id + ) + ) + # self.cursor.execute("SELECT result FROM inference WHERE picture_id=%s AND model_id=%s",(picture_id,model_id,)) + self.assertTrue(validator.is_valid_uuid(result["inference_id"])) + + def test_upload_picture_known(self): + """ + Test the upload picture function with a known seed + """ + picture_set_id = asyncio.run( + datastore.create_picture_set( + self.cursor, self.container_client, 0, self.user_id + ) + ) + picture_id = asyncio.run( + Nachet.upload_picture_known( + self.cursor, + self.user_id, + self.pic_encoded, + self.container_client, + self.seed_id, + picture_set_id, + ) + ) + self.assertTrue(validator.is_valid_uuid(picture_id)) + + def test_upload_picture_known_error_user_not_found(self): + """ + This test checks if the upload_picture_known function correctly raise an exception if the user given doesn't exist in db + """ + picture_set_id = asyncio.run( + datastore.create_picture_set( + self.cursor, self.container_client, 0, self.user_id + ) + ) + with self.assertRaises(Nachet.user.UserNotFoundError): + asyncio.run( + Nachet.upload_picture_known( + self.cursor, + uuid.uuid4(), + self.pic_encoded, + self.container_client, + self.seed_id, + picture_set_id, + ) + ) + + def test_upload_picture_known_connection_error(self): + """ + This test checks if the upload_picture_known function correctly raise an exception if the connection to the db fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + picture_set_id = asyncio.run( + datastore.create_picture_set( + self.cursor, self.container_client, 0, self.user_id + ) + ) + with self.assertRaises(Exception): + asyncio.run( + Nachet.upload_picture_known( + mock_cursor, + self.user_id, + self.pic_encoded, + self.container_client, + self.seed_id, + picture_set_id, + ) + ) + + +class test_picture_set(unittest.TestCase): + def setUp(self): + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = self.con.cursor() + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + self.connection_str = BLOB_CONNECTION_STRING + self.user_email = "test@email" + self.user_obj = asyncio.run( + datastore.new_user( + self.cursor, self.user_email, self.connection_str, "test-user" + ) + ) + self.image = Image.new("RGB", (1980, 1080), "blue") + self.image_byte_array = io.BytesIO() + self.image.save(self.image_byte_array, format="TIFF") + self.pic_encoded = self.image.tobytes() + # self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) + self.container_name = "test-container" + self.user_id = datastore.User.get_id(self.user_obj) + self.container_client = asyncio.run( + datastore.get_user_container_client( + self.user_id, + BLOB_CONNECTION_STRING, + BLOB_ACCOUNT, + BLOB_KEY, + "test-user", + ) + ) + self.seed_name = "test-name" + self.seed_id = seed_query.new_seed(self.cursor, self.seed_name) + self.folder_name = "test_folder" + self.picture_set_id = asyncio.run( + datastore.create_picture_set( + self.cursor, self.container_client, 0, self.user_id, self.folder_name + ) + ) + self.pictures_ids = asyncio.run( + datastore.upload_pictures( + self.cursor, + self.user_id, + [self.pic_encoded, self.pic_encoded, self.pic_encoded], + self.container_client, + self.picture_set_id, + ) + ) + with open("tests/Nachet/inference_result.json") as file: + self.inference = json.load(file) + + self.dev_user_id = datastore.user.get_user_id( + self.cursor, os.environ["DEV_USER_EMAIL"] + ) + self.dev_container_client = asyncio.run( + datastore.get_user_container_client( + self.dev_user_id, BLOB_CONNECTION_STRING, BLOB_ACCOUNT, BLOB_KEY + ) + ) + + def tearDown(self): + self.con.rollback() + self.container_client.delete_container() + db.end_query(self.con, self.cursor) + + def test_find_validated_pictures(self): + """ + This test checks if the find_validated_pictures function correctly returns the validated pictures of a picture_set + """ + + self.assertEqual( + len( + asyncio.run( + Nachet.find_validated_pictures( + self.cursor, str(self.user_id), str(self.picture_set_id) + ) + ) + ), + 0, + "No validated pictures should be found", + ) + + inferences = [] + for picture_id in self.pictures_ids: + # Using deepcopy to ensure each inference is a unique object without shared references + inference_copy = deepcopy(self.inference) + inference = asyncio.run( + Nachet.register_inference_result( + self.cursor, + self.user_id, + inference_copy, + picture_id, + "test_model_id", + ) + ) + inferences.append(inference) + + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, + inferences[0]["inference_id"], + self.user_id, + [box["box_id"] for box in inferences[0]["boxes"]], + ) + ) + + self.assertEqual( + len( + asyncio.run( + Nachet.find_validated_pictures( + self.cursor, str(self.user_id), str(self.picture_set_id) + ) + ) + ), + 1, + "One validated pictures should be found", + ) + + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, + inferences[1]["inference_id"], + self.user_id, + [box["box_id"] for box in inferences[1]["boxes"]], + ) + ) + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, + inferences[2]["inference_id"], + self.user_id, + [box["box_id"] for box in inferences[2]["boxes"]], + ) + ) + + self.assertEqual( + len( + asyncio.run( + Nachet.find_validated_pictures( + self.cursor, str(self.user_id), str(self.picture_set_id) + ) + ) + ), + 3, + "One validated pictures should be found", + ) + + def test_find_validated_pictures_error_user_not_found(self): + """ + This test checks if the find_validated_pictures function correctly raise an exception if the user given doesn't exist in db + """ + pictures_id = [] + for i in range(3): + picture_id = asyncio.run( + Nachet.upload_picture_unknown( + self.cursor, + self.user_id, + self.pic_encoded, + self.container_client, + self.picture_set_id, + ) + ) + pictures_id.append(picture_id) + + with self.assertRaises(Nachet.user.UserNotFoundError): + asyncio.run( + Nachet.find_validated_pictures( + self.cursor, str(uuid.uuid4()), str(self.picture_set_id) + ) + ) + + def test_find_validated_pictures_error_connection_error(self): + """ + This test checks if the find_validated_pictures function correctly raise an exception if the connection to the db fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + asyncio.run( + Nachet.find_validated_pictures( + mock_cursor, str(self.user_id), str(self.picture_set_id) + ) + ) + + def test_find_validated_pictures_error_picture_set_not_found(self): + """ + This test checks if the find_validated_pictures function correctly raise an exception if the picture set given doesn't exist in db + """ + with self.assertRaises(Nachet.picture.PictureSetNotFoundError): + asyncio.run( + Nachet.find_validated_pictures( + self.cursor, str(self.user_id), str(uuid.uuid4()) + ) + ) + + def test_find_validated_pictures_error_not_owner(self): + """ + This test checks if the find_validated_pictures function correctly raise an exception if the user is not the owner of the picture set + """ + not_owner_user_obj = asyncio.run( + datastore.new_user( + self.cursor, "notowner@email", self.connection_str, "test-user" + ) + ) + not_owner_user_id = datastore.User.get_id(not_owner_user_obj) + + with self.assertRaises(Nachet.UserNotOwnerError): + asyncio.run( + Nachet.find_validated_pictures( + self.cursor, str(not_owner_user_id), str(self.picture_set_id) + ) + ) + + container_client = asyncio.run( + datastore.get_user_container_client( + not_owner_user_id, + BLOB_CONNECTION_STRING, + BLOB_ACCOUNT, + BLOB_KEY, + "test-user", + ) + ) + container_client.delete_container() + + def test_delete_picture_set_with_archive(self): + """ + This test checks if the delete_picture_set_with_archive function correctly archive the picture set in dev container and delete it from user container + """ + # Create inferences for pictures in the picture set + inferences = [] + for picture_id in self.pictures_ids: + # Using deepcopy to ensure each inference is a unique object without shared references + inference_copy = deepcopy(self.inference) + inference = asyncio.run( + Nachet.register_inference_result( + self.cursor, + self.user_id, + inference_copy, + picture_id, + "test_model_id", + ) + ) + inferences.append(inference) + # Validate 2 of 3 pictures in the picture set + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, + inferences[1]["inference_id"], + self.user_id, + [box["box_id"] for box in inferences[1]["boxes"]], + ) + ) + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, + inferences[2]["inference_id"], + self.user_id, + [box["box_id"] for box in inferences[2]["boxes"]], + ) + ) + validated_pictures = asyncio.run( + Nachet.find_validated_pictures( + self.cursor, str(self.user_id), str(self.picture_set_id) + ) + ) + + dev_nb_folders = len( + asyncio.run(datastore.get_picture_sets_info(self.cursor, self.dev_user_id)) + ) + # Check there is the right number of picture sets in db for each user + self.assertEqual( + len( + asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) + ), + 2, + ) + + dev_picture_set_id = asyncio.run( + Nachet.delete_picture_set_with_archive( + self.cursor, + str(self.user_id), + str(self.picture_set_id), + self.container_client, + ) + ) + # Check there is the right number of picture sets in db for each user after moving + self.assertEqual( + len( + asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) + ), + 1, + ) + self.assertEqual( + len( + asyncio.run( + datastore.get_picture_sets_info(self.cursor, self.dev_user_id) + ) + ), + dev_nb_folders + 1, + ) + + # Check blobs have also moved in blob storage + for picture_id in validated_pictures: + blob_name = "{}/{}.png".format(self.folder_name, str(picture_id)) + with self.assertRaises(Exception): + asyncio.run( + datastore.azure_storage.get_blob(self.container_client, blob_name) + ) + + blob_name = "{}/{}/{}.png".format( + str(self.user_id), self.folder_name, str(picture_id) + ) + blob = asyncio.run( + datastore.azure_storage.get_blob(self.dev_container_client, blob_name) + ) + self.assertEqual(blob, self.pic_encoded) + + # TEAR DOWN + # Delete the user folder in the blob storage + asyncio.run( + datastore.azure_storage.delete_folder( + self.dev_container_client, str(dev_picture_set_id) + ) + ) + asyncio.run( + datastore.azure_storage.delete_folder( + self.dev_container_client, str(self.user_id) + ) + ) + + def test_delete_picture_set_with_archive_error_user_not_found(self): + """ + This test checks if the delete_picture_set_with_archive function correctly raise an exception if the user given doesn't exist in db + """ + with self.assertRaises(Nachet.user.UserNotFoundError): + asyncio.run( + Nachet.delete_picture_set_with_archive( + self.cursor, + str(uuid.uuid4()), + str(self.picture_set_id), + self.container_client, + ) + ) + + def test_delete_picture_set_with_archive_error_connection_error(self): + """ + This test checks if the delete_picture_set_with_archive function correctly raise an exception if the connection to the db fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + asyncio.run( + Nachet.delete_picture_set_with_archive( + mock_cursor, + str(self.user_id), + str(self.picture_set_id), + self.container_client, + ) + ) + + def test_delete_picture_set_with_archive_error_picture_set_not_found(self): + """ + This test checks if the delete_picture_set_with_archive function correctly raise an exception if the picture set given doesn't exist in db + """ + with self.assertRaises(Nachet.picture.PictureSetNotFoundError): + asyncio.run( + Nachet.delete_picture_set_with_archive( + self.cursor, + str(self.user_id), + str(uuid.uuid4()), + self.container_client, + ) + ) + + def test_delete_picture_set_with_archive_error_not_owner(self): + """ + This test checks if the delete_picture_set_with_archive function correctly raise an exception if the user is not the owner of the picture set + """ + not_owner_user_obj = asyncio.run( + datastore.new_user( + self.cursor, "notowner@email", self.connection_str, "test-user" + ) + ) + not_owner_user_id = datastore.User.get_id(not_owner_user_obj) + + with self.assertRaises(Nachet.UserNotOwnerError): + asyncio.run( + Nachet.delete_picture_set_with_archive( + self.cursor, + str(not_owner_user_id), + str(self.picture_set_id), + self.container_client, + ) + ) + + container_client = asyncio.run( + datastore.get_user_container_client( + not_owner_user_id, + BLOB_CONNECTION_STRING, + BLOB_ACCOUNT, + BLOB_KEY, + "test-user", + ) + ) + container_client.delete_container() + + def test_delete_picture_set_with_archive_error_default_folder(self): + """ + This test checks if the delete_picture_set_with_archive function correctly raise an exception if the user want to delete the folder "General" + """ + general_folder_id = datastore.user.get_default_picture_set( + self.cursor, self.user_id + ) + with self.assertRaises(Nachet.picture.PictureSetDeleteError): + asyncio.run( + Nachet.delete_picture_set_with_archive( + self.cursor, + str(self.user_id), + str(general_folder_id), + self.container_client, + ) + ) + + +class test_feedback(unittest.TestCase): + def setUp(self): + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = self.con.cursor() + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + self.connection_str = BLOB_CONNECTION_STRING + self.user_email = "test@email" + self.user_obj = asyncio.run( + datastore.new_user( + self.cursor, self.user_email, self.connection_str, "test-user" + ) + ) + self.image = Image.new("RGB", (1980, 1080), "blue") + self.image_byte_array = io.BytesIO() + self.image.save(self.image_byte_array, format="TIFF") + self.pic_encoded = self.image.tobytes() + self.container_name = "test-container" + self.user_id = datastore.User.get_id(self.user_obj) + self.container_client = asyncio.run( + datastore.get_user_container_client( + self.user_id, + BLOB_CONNECTION_STRING, + BLOB_ACCOUNT, + BLOB_KEY, + "test-user", + ) + ) + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "inference_result.json") + with open(file_path) as file: + self.inference = json.load(file) + picture_id = asyncio.run( + Nachet.upload_picture_unknown( + self.cursor, self.user_id, self.pic_encoded, self.container_client + ) + ) + model_id = "test_model_id" + self.registered_inference = asyncio.run( + Nachet.register_inference_result( + self.cursor, self.user_id, self.inference, picture_id, model_id + ) + ) + self.registered_inference["user_id"] = self.user_id + self.mock_box = {"topX": 123, "topY": 456, "bottomX": 789, "bottomY": 123} + self.inference_id = self.registered_inference.get("inference_id") + self.boxes_id = [] + self.top_id = [] + self.unreal_seed_id = Nachet.seed.new_seed(self.cursor, "unreal_seed") + for box in self.registered_inference["boxes"]: + self.boxes_id.append(box["box_id"]) + self.top_id.append(box["top_id"]) + box["classId"] = Nachet.seed.get_seed_id(self.cursor, box["label"]) + + def tearDown(self): + self.con.rollback() + self.container_client.delete_container() + db.end_query(self.con, self.cursor) + + def test_new_perfect_inference_feedback(self): + """ + This test checks if the new_perfect_inference_feeback function correctly updates the inference object after a perfect feedback is given + """ + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, self.inference_id, self.user_id, self.boxes_id + ) + ) + for i in range(len(self.boxes_id)): + object = Nachet.inference.get_inference_object( + self.cursor, self.boxes_id[i] + ) + # verified_id must be equal to top_id + self.assertEqual(str(object[4]), self.top_id[i]) + # valid column must be true + self.assertTrue(object[5]) + + def test_new_perfect_inference_feedback_error_verified_inference(self): + """ + This test checks if the new_perfect_inference_feeback function correctly raise an exception if the inference given is already verified + """ + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, self.inference_id, self.user_id, self.boxes_id + ) + ) + self.assertTrue( + Nachet.inference.is_inference_verified(self.cursor, self.inference_id) + ) + with self.assertRaises(Nachet.inference.InferenceAlreadyVerifiedError): + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, self.inference_id, self.user_id, self.boxes_id + ) + ) + + def test_new_perfect_inference_feedback_error_inference_not_found(self): + """ + This test checks if the new_perfect_inference_feeback function correctly raise an exception if the inference given doesn't exist in db + """ + with self.assertRaises(Nachet.inference.InferenceNotFoundError): + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, str(uuid.uuid4()), self.user_id, self.boxes_id + ) + ) + + def test_new_perfect_inference_feedback_error_inference_object_not_found(self): + """ + This test checks if the new_perfect_inference_feeback function correctly raise an exception if one of the inference object given doesn't exist in db + """ + with self.assertRaises(Nachet.inference.InferenceObjectNotFoundError): + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, + self.inference_id, + self.user_id, + [self.boxes_id[0], str(uuid.uuid4())], + ) + ) + + def test_new_perfect_inference_feedback_error_user_not_found(self): + """ + This test checks if the new_perfect_inference_feeback function correctly raise an exception if the user given doesn't exist in db + """ + with self.assertRaises(Nachet.user.UserNotFoundError): + asyncio.run( + Nachet.new_perfect_inference_feeback( + self.cursor, self.inference_id, str(uuid.uuid4()), self.boxes_id + ) + ) + + def test_new_perfect_inference_feedback_connection_error(self): + """ + This test checks if the new_perfect_inference_feeback function correctly raise an exception if the connection to the db fails + """ + mock_cursor = MagicMock() + mock_cursor.fetchone.side_effect = Exception("Connection error") + with self.assertRaises(Exception): + asyncio.run( + Nachet.new_perfect_inference_feeback( + mock_cursor, self.inference_id, self.user_id, self.boxes_id + ) + ) + + def test_new_correction_inference_feedback(self): + """ + This test checks if the new_correction_inference_feeback function correctly + """ + self.assertTrue(validator.is_valid_uuid(self.inference_id)) + + asyncio.run( + Nachet.new_correction_inference_feedback( + self.cursor, self.registered_inference, 1 + ) + ) + for i in range(len(self.boxes_id)): + object = Nachet.inference.get_inference_object( + self.cursor, self.boxes_id[i] + ) + # verified_id must be equal to top_id + self.assertEqual(str(object[4]), self.top_id[i]) + # valid column must be true + self.assertTrue(object[6]) + + def test_new_correction_inference_feedback_new_guess(self): + """ + This test checks if the new_correction_inference_feeback function correctly when another guess is verified + """ + self.assertTrue(validator.is_valid_uuid(self.inference_id)) + new_top_ids = [] + for box in self.registered_inference["boxes"]: + box["label"] = box["topN"][1]["label"] + box["classId"] = Nachet.seed.get_seed_id(self.cursor, box["label"]) + new_top_ids.append(box["topN"][1]["object_id"]) + asyncio.run( + Nachet.new_correction_inference_feedback( + self.cursor, self.registered_inference, 1 + ) + ) + for i in range(len(self.boxes_id)): + object_db = Nachet.inference.get_inference_object( + self.cursor, self.boxes_id[i] + ) + # verified_id must be equal to top_id + self.assertTrue(str(object_db[4]) == new_top_ids[i]) + # valid column must be true + self.assertTrue(object_db[6]) + + def test_new_correction_inference_feedback_box_edited(self): + """ + This test checks if the new_correction_inference_feeback function correctly when the box metadata is updated + """ + self.assertTrue(validator.is_valid_uuid(self.inference_id)) + for box in self.registered_inference["boxes"]: + box["box"] = self.mock_box + asyncio.run( + Nachet.new_correction_inference_feedback( + self.cursor, self.registered_inference, 1 + ) + ) + for box in self.registered_inference["boxes"]: + object_db = Nachet.inference.get_inference_object( + self.cursor, box["box_id"] + ) + # The new box metadata must be updated + self.assertDictEqual(object_db[1], self.mock_box) + # The top_id must be equal to the previous top_id + self.assertEqual(str(object_db[4]), box["top_id"]) + # valid column must be true + self.assertTrue(object_db[6]) + + def test_new_correction_inference_feedback_not_guess(self): + """ + This test checks if the new_correction_inference_feeback function correctly when the box is not a guess + """ + self.assertTrue(validator.is_valid_uuid(self.inference_id)) + for box in self.registered_inference["boxes"]: + box["label"] = "unreal_seed" + box["classId"] = self.unreal_seed_id + asyncio.run( + Nachet.new_correction_inference_feedback( + self.cursor, self.registered_inference, 1 + ) + ) + for i in range(len(self.boxes_id)): + object_db = Nachet.inference.get_inference_object( + self.cursor, self.boxes_id[i] + ) + # verified_id must be equal to the new_top_id + new_top_id = Nachet.inference.get_seed_object_id( + self.cursor, self.unreal_seed_id, object_db[0] + ) + self.assertTrue(validator.is_valid_uuid(new_top_id)) + self.assertEqual(str(object_db[4]), str(new_top_id)) + # valid column must be true + self.assertTrue(object_db[6]) + + def test_new_correction_inference_feedback_not_valid(self): + """ + This test checks if the new_correction_inference_feeback function correctly when the box is not a guess + """ + self.assertTrue(validator.is_valid_uuid(self.inference_id)) + for box in self.registered_inference["boxes"]: + box["label"] = "" + box["classId"] = "" + asyncio.run( + Nachet.new_correction_inference_feedback( + self.cursor, self.registered_inference, 1 + ) + ) + for i in range(len(self.boxes_id)): + object_db = Nachet.inference.get_inference_object( + self.cursor, self.boxes_id[i] + ) + # verified_id must not be an id + self.assertEqual(object_db[4], None) + # valid column must be false + self.assertFalse(object_db[6]) + + def test_new_correction_inference_feedback_unknown_seed(self): + """ + This test checks if the new_correction_inference_feeback function correctly when the box is not a guess + """ + self.assertTrue(validator.is_valid_uuid(self.inference_id)) + for box in self.registered_inference["boxes"]: + box["label"] = "unknown_seed" + box["classId"] = "" + asyncio.run( + Nachet.new_correction_inference_feedback( + self.cursor, self.registered_inference, 1 + ) + ) + for i in range(len(self.boxes_id)): + object_db = Nachet.inference.get_inference_object( + self.cursor, self.boxes_id[i] + ) + # verified_id must be equal to an id + self.assertTrue(validator.is_valid_uuid(str(object_db[4]))) + # valid column must be true + self.assertTrue(object_db[6]) diff --git a/tests/test_mass_import.py b/tests/testBin_mass_import.py similarity index 100% rename from tests/test_mass_import.py rename to tests/testBin_mass_import.py diff --git a/tests/test_upload_picture_set.py b/tests/testBin_upload_picture_set.py similarity index 100% rename from tests/test_upload_picture_set.py rename to tests/testBin_upload_picture_set.py diff --git a/tests/test_azure_storage.py b/tests/test_azure_storage.py index acab6d69..0949e490 100644 --- a/tests/test_azure_storage.py +++ b/tests/test_azure_storage.py @@ -23,8 +23,9 @@ get_folder_uuid, get_blobs_from_tag, get_directories, - move_blob + move_blob, ) + BLOB_CONNECTION_STRING = os.environ["NACHET_STORAGE_URL_TESTING"] if BLOB_CONNECTION_STRING is None or BLOB_CONNECTION_STRING == "": raise ValueError("NACHET_STORAGE_URL_TESTING is not set") @@ -37,19 +38,20 @@ if BLOB_KEY is None or BLOB_KEY == "": raise ValueError("NACHET_BLOB_KEY is not set") + class TestMountContainerFunction(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING self.account_name = BLOB_ACCOUNT self.account_key = BLOB_KEY - self.credential= blob.get_account_sas(self.account_name, self.account_key) - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - + self.credential = blob.get_account_sas(self.account_name, self.account_key) + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + def test_mount_existing_container(self): container_client = asyncio.run( - mount_container(self.storage_url, self.container_uuid,True,self.tier) + mount_container(self.storage_url, self.container_uuid, True, self.tier) ) self.assertTrue(container_client.exists()) @@ -60,49 +62,52 @@ def test_mount_nonexisting_container_create(self): tests when a container does not exists and create_container flag is set to True, should create a new container and return the container client """ - not_uuid="notuuid" + not_uuid = "notuuid" container_client = asyncio.run( - mount_container(self.storage_url, not_uuid,True,self.tier,self.credential) + mount_container( + self.storage_url, not_uuid, True, self.tier, self.credential + ) ) self.assertTrue(container_client.exists()) container_client.delete_container() def test_mount_nonexisting_container_no_create(self): - not_uuid="alsonotuuid" + not_uuid = "alsonotuuid" with self.assertRaises(MountContainerError): asyncio.run( - mount_container(self.storage_url, not_uuid,False,self.tier,self.credential) + mount_container( + self.storage_url, not_uuid, False, self.tier, self.credential + ) ) - + def test_mount_container_connection_string_error(self): with self.assertRaises(ConnectionStringError): asyncio.run( - mount_container("invalid-url", self.container_uuid,True,self.tier) + mount_container("invalid-url", self.container_uuid, True, self.tier) ) - class TestGetBlob(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - self.container_name=f"{self.tier}-{self.container_uuid}" + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + self.container_name = f"{self.tier}-{self.container_uuid}" self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.container_client = self.blob_service_client.create_container(self.container_name) + self.container_client = self.blob_service_client.create_container( + self.container_name + ) self.blob_name = "test_blob" - self.blob= "test_blob_content" + self.blob = "test_blob_content" self.container_client.upload_blob(name=self.blob_name, data=self.blob) - + def tearDown(self): self.container_client.delete_container() - + def test_get_blob(self): self.assertTrue(self.container_client.exists()) - result = asyncio.run( - get_blob(self.container_client, self.blob_name) - ) + result = asyncio.run(get_blob(self.container_client, self.blob_name)) self.assertEqual(str(result.decode()), self.blob) @@ -121,29 +126,37 @@ def test_get_blob_error(self): class TestUploadImage(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - self.container_name=f"{self.tier}-{self.container_uuid}" + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + self.container_name = f"{self.tier}-{self.container_uuid}" self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.container_client = self.blob_service_client.create_container(self.container_name) + self.container_client = self.blob_service_client.create_container( + self.container_name + ) self.image = Image.new("RGB", (1980, 1080), "blue") self.image_byte_array = io.BytesIO() self.image.save(self.image_byte_array, format="TIFF") self.image_byte = self.image.tobytes() self.image_hash = asyncio.run(generate_hash(self.image_byte)) self.image_uuid = str(uuid.uuid4()) - #self.container_client.upload_blob(name=self.blob_name, data=self.blob) - self.folder_name="test_folder" + # self.container_client.upload_blob(name=self.blob_name, data=self.blob) + self.folder_name = "test_folder" self.folder_uuid = str(uuid.uuid4()) asyncio.run(create_folder(self.container_client, self.folder_name)) - + def tearDown(self): self.container_client.delete_container() - + def test_upload_image(self): expected_result = "{}/{}.png".format(self.folder_name, self.image_uuid) result = asyncio.run( - upload_image(self.container_client, self.folder_name,self.folder_uuid,self.image_hash,self.image_uuid) + upload_image( + self.container_client, + self.folder_name, + self.folder_uuid, + self.image_hash, + self.image_uuid, + ) ) self.assertEqual(result, expected_result) @@ -152,26 +165,35 @@ def test_upload_image_wrong_folder(self): """ This test checkes if the function raises an error when the folder does not exist """ - not_folder_name="not_folder" + not_folder_name = "not_folder" with self.assertRaises(CreateDirectoryError): asyncio.run( - upload_image(self.container_client, not_folder_name,str(uuid.uuid4()),self.image_hash,self.image_uuid) + upload_image( + self.container_client, + not_folder_name, + str(uuid.uuid4()), + self.image_hash, + self.image_uuid, + ) ) + class TestIsAFolder(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - self.container_name=f"{self.tier}-{self.container_uuid}" + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + self.container_name = f"{self.tier}-{self.container_uuid}" self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.container_client = self.blob_service_client.create_container(self.container_name) - self.folder_name="test_folder" + self.container_client = self.blob_service_client.create_container( + self.container_name + ) + self.folder_name = "test_folder" asyncio.run(create_folder(self.container_client, self.folder_name)) - + def tearDown(self): self.container_client.delete_container() - + def test_is_a_folder(self): self.assertTrue(self.container_client.exists()) result = asyncio.run(is_a_folder(self.container_client, self.folder_name)) @@ -179,41 +201,46 @@ def test_is_a_folder(self): def test_is_not_a_folder(self): self.assertTrue(self.container_client.exists()) - not_folder_name="not_folder" + not_folder_name = "not_folder" result = asyncio.run(is_a_folder(self.container_client, not_folder_name)) self.assertFalse(result) - + + class TestCreateFolder(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - self.container_name=f"{self.tier}-{self.container_uuid}" + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + self.container_name = f"{self.tier}-{self.container_uuid}" self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.container_client = self.blob_service_client.create_container(self.container_name) - + self.container_client = self.blob_service_client.create_container( + self.container_name + ) + def tearDown(self): self.container_client.delete_container() - + def test_create_folder(self): - folder_name="test_folder" - result = asyncio.run(create_folder(self.container_client, folder_name)) + folder_name = "test_folder" + result = asyncio.run(create_folder(self.container_client, folder_name)) self.assertTrue(result) def test_create_existing_folder(self): - folder_name="test_folder" + folder_name = "test_folder" asyncio.run(create_folder(self.container_client, folder_name)) with self.assertRaises(CreateDirectoryError): asyncio.run(create_folder(self.container_client, folder_name)) - def test_create_folder_error(self): - folder_name="tessst_folder" + folder_name = "tessst_folder" mock_container_client = Mock() - mock_container_client.list_blobs.side_effect = FolderListError("Resource not found") + mock_container_client.list_blobs.side_effect = FolderListError( + "Resource not found" + ) with self.assertRaises(CreateDirectoryError): asyncio.run(create_folder(mock_container_client, folder_name)) - + + class TestGenerateHash(unittest.TestCase): def setUp(self): self.image = Image.new("RGB", (1980, 1080), "blue") @@ -228,55 +255,73 @@ def test_generate_hash(self): def test_generate_hash_error(self): with self.assertRaises(GenerateHashError): asyncio.run(generate_hash("not an image")) - + + class TestGetFolderUUID(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - self.container_name=f"{self.tier}-{self.container_uuid}" + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + self.container_name = f"{self.tier}-{self.container_uuid}" self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.container_client = self.blob_service_client.create_container(self.container_name) - self.folder_name="test_folder" - self.folder_uuid= str(uuid.uuid4()) - asyncio.run(create_folder(self.container_client,self.folder_uuid, self.folder_name)) - + self.container_client = self.blob_service_client.create_container( + self.container_name + ) + self.folder_name = "test_folder" + self.folder_uuid = str(uuid.uuid4()) + asyncio.run( + create_folder(self.container_client, self.folder_uuid, self.folder_name) + ) + def tearDown(self): self.container_client.delete_container() - 0 + def test_get_folder_uuid(self): - result = asyncio.run(get_folder_uuid(self.container_client,self.folder_name)) + result = asyncio.run(get_folder_uuid(self.container_client, self.folder_name)) self.assertEqual(result, self.folder_uuid) def test_get_folder_uuid_error(self): - not_folder_name="not_folder" + not_folder_name = "not_folder" with self.assertRaises(GetFolderUUIDError): - asyncio.run(get_folder_uuid(self.container_client,not_folder_name)) + asyncio.run(get_folder_uuid(self.container_client, not_folder_name)) + -class TestGetBlobsFromTag(unittest.TestCase) : +class TestGetBlobsFromTag(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - self.container_name=f"{self.tier}-{self.container_uuid}" + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + self.container_name = f"{self.tier}-{self.container_uuid}" self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.container_client = self.blob_service_client.create_container(self.container_name) + self.container_client = self.blob_service_client.create_container( + self.container_name + ) self.image = Image.new("RGB", (1980, 1080), "blue") self.image_byte_array = io.BytesIO() self.image.save(self.image_byte_array, format="TIFF") self.image_byte = self.image.tobytes() self.image_hash = asyncio.run(generate_hash(self.image_byte)) self.image_uuid = str(uuid.uuid4()) - self.folder_name="test_folder" - self.folder_uuid= str(uuid.uuid4()) - asyncio.run(create_folder(self.container_client,self.folder_uuid, self.folder_name)) - asyncio.run(upload_image(self.container_client, self.folder_name,self.folder_uuid,self.image_hash,self.image_uuid)) - + self.folder_name = "test_folder" + self.folder_uuid = str(uuid.uuid4()) + asyncio.run( + create_folder(self.container_client, self.folder_uuid, self.folder_name) + ) + asyncio.run( + upload_image( + self.container_client, + self.folder_name, + self.folder_uuid, + self.image_hash, + self.image_uuid, + ) + ) + def tearDown(self): self.container_client.delete_container() def test_get_blobs_from_tag(self): - tag = 'test_folder' + tag = "test_folder" try: result = asyncio.run(get_blobs_from_tag(self.container_client, tag)) self.assertGreater(len(result), 0) @@ -287,92 +332,137 @@ def test_get_blobs_from_tag_error(self): tag = "wrong_tag" with self.assertRaises(Exception): asyncio.run(get_blobs_from_tag(self.container_client, tag)) - - -class TestGetDirectories(unittest.TestCase) : - def setUp(self) : + +class TestGetDirectories(unittest.TestCase): + def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - self.tier="testuser" - self.container_uuid=str(uuid.uuid4()) - self.container_name=f"{self.tier}-{self.container_uuid}" + self.tier = "testuser" + self.container_uuid = str(uuid.uuid4()) + self.container_name = f"{self.tier}-{self.container_uuid}" self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.container_client = self.blob_service_client.create_container(self.container_name) - self.folder_name="test_folder" - self.folder_uuid= str(uuid.uuid4()) + self.container_client = self.blob_service_client.create_container( + self.container_name + ) + self.folder_name = "test_folder" + self.folder_uuid = str(uuid.uuid4()) self.image = Image.new("RGB", (1980, 1080), "blue") self.image_byte_array = io.BytesIO() self.image.save(self.image_byte_array, format="TIFF") self.image_byte = self.image.tobytes() self.image_hash = asyncio.run(generate_hash(self.image_byte)) self.image_uuid = str(uuid.uuid4()) - asyncio.run(create_folder(self.container_client,self.folder_uuid, self.folder_name)) - + asyncio.run( + create_folder(self.container_client, self.folder_uuid, self.folder_name) + ) + def tearDown(self): self.container_client.delete_container() - - def test_get_directories(self) : + + def test_get_directories(self): try: result = asyncio.run(get_directories(self.container_client)) self.assertEqual(len(result), 1) self.assertEqual(result.get(self.folder_name), 0) - - asyncio.run(upload_image(self.container_client, self.folder_name,self.image_hash,self.image_uuid)) - + + asyncio.run( + upload_image( + self.container_client, + self.folder_name, + self.image_hash, + self.image_uuid, + ) + ) + result = asyncio.run(get_directories(self.container_client)) self.assertEqual(len(result), 1) self.assertEqual(result.get(self.folder_name), 1) except Exception as e: print(f"Test failed with exception: {e}") - + def test_get_directories_error(self): mock_container_client = Mock() - mock_container_client.upload_blob.side_effect = FolderListError("Resource not found") + mock_container_client.upload_blob.side_effect = FolderListError( + "Resource not found" + ) with self.assertRaises(FolderListError): asyncio.run(get_directories(mock_container_client)) - + + class TestMoveBlob(unittest.TestCase): def setUp(self): self.storage_url = os.environ.get("NACHET_STORAGE_URL") self.blob_service_client = blob.create_BlobServiceClient(self.storage_url) - self.tier="testuser" - - self.container_uuid_source=str(uuid.uuid4()) - self.container_name_source=f"{self.tier}-{self.container_uuid_source}" - self.container_client_source = self.blob_service_client.create_container(self.container_name_source) - - self.container_uuid_dest=str(uuid.uuid4()) - self.container_name_dest=f"{self.tier}-{self.container_uuid_dest}" - self.container_client_dest = self.blob_service_client.create_container(self.container_name_dest) - + self.tier = "testuser" + + self.container_uuid_source = str(uuid.uuid4()) + self.container_name_source = f"{self.tier}-{self.container_uuid_source}" + self.container_client_source = self.blob_service_client.create_container( + self.container_name_source + ) + + self.container_uuid_dest = str(uuid.uuid4()) + self.container_name_dest = f"{self.tier}-{self.container_uuid_dest}" + self.container_client_dest = self.blob_service_client.create_container( + self.container_name_dest + ) + self.blob_name = "test_blob" - self.blob= "test_blob_content" + self.blob = "test_blob_content" self.container_client_source.upload_blob(name=self.blob_name, data=self.blob) def tearDown(self): self.container_client_source.delete_container() self.container_client_dest.delete_container() - + def test_move_blob(self): - asyncio.run(move_blob(self.blob_name, self.blob_name, None, self.container_client_source, self.container_client_dest)) - + asyncio.run( + move_blob( + self.blob_name, + self.blob_name, + None, + self.container_client_source, + self.container_client_dest, + ) + ) + with self.assertRaises(Exception): asyncio.run(get_blob(self.container_client_source, self.blob_name)) - + blob = asyncio.run(get_blob(self.container_client_dest, self.blob_name)) self.assertEqual(str(blob.decode()), self.blob) - + def test_move_blob_error(self): mock_container_client_source = Mock() - mock_container_client_source.get_blob_client.side_effect = Exception("Resource not found") + mock_container_client_source.get_blob_client.side_effect = Exception( + "Resource not found" + ) with self.assertRaises(Exception): - asyncio.run(move_blob(self.blob_name, self.blob_name, None, mock_container_client_source, self.container_client_dest)) - - + asyncio.run( + move_blob( + self.blob_name, + self.blob_name, + None, + mock_container_client_source, + self.container_client_dest, + ) + ) + mock_container_client_dest = Mock() - mock_container_client_dest.get_blob_client.side_effect = Exception("Get blob Error") + mock_container_client_dest.get_blob_client.side_effect = Exception( + "Get blob Error" + ) with self.assertRaises(Exception): - asyncio.run(move_blob(self.blob_name, self.blob_name, None, self.container_client_source, mock_container_client_dest)) - + asyncio.run( + move_blob( + self.blob_name, + self.blob_name, + None, + self.container_client_source, + mock_container_client_dest, + ) + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_blob.py b/tests/test_blob.py index 60d1f1cb..e588bb90 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -10,14 +10,15 @@ if BLOB_CONNECTION_STRING is None or BLOB_CONNECTION_STRING == "": raise ValueError("NACHET_STORAGE_URL_TESTING is not set") + class TestGetBlobServiceClient(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING - + def test_create_BlobServiceClient(self): result = blob.create_BlobServiceClient(self.storage_url) - + self.assertGreater(len(result.api_version), 0) def test_get_blob_service_unsuccessful(self): @@ -30,10 +31,14 @@ class TestCreateContainerClient(unittest.TestCase): def setUp(self): self.storage_url = BLOB_CONNECTION_STRING self.container_name = str(uuid.uuid4()) - self.blob_service_client = BlobServiceClient.from_connection_string(self.storage_url) - + self.blob_service_client = BlobServiceClient.from_connection_string( + self.storage_url + ) + def test_create_container_client(self): - result = blob.create_container_client(self.blob_service_client, self.container_name) + result = blob.create_container_client( + self.blob_service_client, self.container_name + ) self.assertTrue(result.exists()) self.assertGreater(len(result.container_name), 0) diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 9e3876f5..3e39c4e0 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -7,21 +7,17 @@ import unittest from unittest.mock import MagicMock, patch from PIL import Image -import json import uuid import asyncio import datastore.db.__init__ as db import datastore.__init__ as datastore import datastore.db.metadata.validator as validator -import datastore.db.queries.seed as seed_query -from copy import deepcopy DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": raise ValueError("NACHET_DB_URL is not set") - DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") if DB_SCHEMA is None or DB_SCHEMA == "": raise ValueError("NACHET_SCHEMA_TESTING is not set") @@ -30,63 +26,17 @@ if BLOB_CONNECTION_STRING is None or BLOB_CONNECTION_STRING == "": raise ValueError("NACHET_STORAGE_URL_TESTING is not set") -class test_ml_structure(unittest.TestCase): - def setUp(self): - with open("tests/ml_structure_exemple.json") as file: - self.ml_dict= json.load(file) - self.con = db.connect_db(DB_CONNECTION_STRING) - self.cursor = self.con.cursor() - db.create_search_path(self.con, self.cursor,DB_SCHEMA) - - def tearDown(self): - self.con.rollback() - db.end_query(self.con, self.cursor) - - - def test_import_ml_structure_from_json(self): - """ - Test the import function. - """ - asyncio.run(datastore.import_ml_structure_from_json_version(self.cursor,self.ml_dict)) - self.cursor.execute("SELECT id FROM model WHERE name='that_model_name'") - model_id=self.cursor.fetchone()[0] - self.assertTrue(validator.is_valid_uuid(str(model_id))) - self.cursor.execute("SELECT id FROM pipeline WHERE name='Second Pipeline'") - pipeline_id=self.cursor.fetchone()[0] - self.assertTrue(validator.is_valid_uuid(str(pipeline_id))) - self.cursor.execute("SELECT id FROM pipeline_model WHERE pipeline_id=%s AND model_id=%s",(pipeline_id,model_id,)) - self.assertTrue(validator.is_valid_uuid(self.cursor.fetchone()[0])) - - def test_get_ml_structure(self): - """ - Test the get function. - """ - - #asyncio.run(datastore.import_ml_structure_from_json_version(self.cursor,self.ml_dict)) - ml_structure= asyncio.run(datastore.get_ml_structure(self.cursor)) - #self.assertDictEqual(ml_structure,self.ml_dict) - for pipeline in self.ml_dict["pipelines"]: - for key in pipeline: - if key!='Accuracy': - self.assertTrue((key in ml_structure["pipelines"][0].keys()),f"Key {key} was not found and expected in the returned dictionary") - for model in self.ml_dict["models"]: - for key in model: - if key!='Accuracy' and key !='endpoint_name': - #print(key) - self.assertTrue((key in ml_structure["models"][0].keys()),f"Key {key} was not found and expected in the returned dictionary") - - def test_get_ml_structure_eeror(self): - """ - Test the get version function. - """ - mock_cursor = MagicMock() - mock_cursor.fetchall.return_value = [] - with self.assertRaises(datastore.MLRetrievalError): - asyncio.run(datastore.get_ml_structure(mock_cursor)) +BLOB_ACCOUNT = os.environ["NACHET_BLOB_ACCOUNT_TESTING"] +if BLOB_ACCOUNT is None or BLOB_ACCOUNT == "": + raise ValueError("NACHET_BLOB_ACCOUNT is not set") + +BLOB_KEY = os.environ["NACHET_BLOB_KEY_TESTING"] +if BLOB_KEY is None or BLOB_KEY == "": + raise ValueError("NACHET_BLOB_KEY is not set") class test_user(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.user_email="test@email" @@ -97,7 +47,7 @@ def setUp(self): def tearDown(self): self.con.rollback() if self.user_id is not None: - self.container_client=asyncio.run(datastore.get_user_container_client(self.user_id,'test-user')) + self.container_client=asyncio.run(datastore.get_user_container_client(self.user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) self.container_client.delete_container() self.user_id=None db.end_query(self.con, self.cursor) @@ -158,12 +108,12 @@ def test_get_user_container_client(self): """ user_obj=asyncio.run(datastore.new_user(self.cursor,self.user_email,self.connection_str,'test-user')) user_id= datastore.User.get_id(user_obj) - container_client=asyncio.run(datastore.get_user_container_client(user_id,'test-user')) + container_client=asyncio.run(datastore.get_user_container_client(user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) self.assertTrue(container_client.exists()) class test_picture(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.connection_str=BLOB_CONNECTION_STRING @@ -176,84 +126,13 @@ def setUp(self): #self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) self.container_name='test-container' self.user_id=datastore.User.get_id(self.user_obj) - self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,'test-user')) - self.seed_name = "test-name" - self.seed_id = seed_query.new_seed(self.cursor, self.seed_name) - with open("tests/inference_result.json") as file: - self.inference= json.load(file) + self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) self.folder_name = "test_folder" def tearDown(self): self.con.rollback() self.container_client.delete_container() db.end_query(self.con, self.cursor) - - def test_upload_picture_unknown(self): - """ - Test the upload picture function. - """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) - self.assertTrue(validator.is_valid_uuid(picture_id)) - - def test_register_inference_result(self): - """ - Test the register inference result function. - """ - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) - model_id = "test_model_id" - - result = asyncio.run(datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, model_id)) - #self.cursor.execute("SELECT result FROM inference WHERE picture_id=%s AND model_id=%s",(picture_id,model_id,)) - self.assertTrue(validator.is_valid_uuid(result["inferenceId"])) - - def test_create_picture_set(self): - """ - Test the creation of a picture set - """ - picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) - self.assertTrue(validator.is_valid_uuid(picture_set_id)) - - def test_create_picture_set_connection_error(self): - """ - This test checks if the create_picture_set function correctly raise an exception if the connection to the db fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - asyncio.run(datastore.create_picture_set(mock_cursor, self.container_client, 0, self.user_id)) - - def test_create_picture_set_error_user_not_found(self): - """ - This test checks if the create_picture_set function correctly raise an exception if the user given doesn't exist in db - """ - with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, uuid.uuid4())) - - def test_upload_picture_known(self): - """ - Test the upload picture function with a known seed - """ - picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) - picture_id = asyncio.run(datastore.upload_picture_known(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.seed_id, picture_set_id)) - self.assertTrue(validator.is_valid_uuid(picture_id)) - - def test_upload_picture_known_error_user_not_found(self): - """ - This test checks if the upload_picture_known function correctly raise an exception if the user given doesn't exist in db - """ - picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) - with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.upload_picture_known(self.cursor, uuid.uuid4(), self.pic_encoded,self.container_client, self.seed_id, picture_set_id)) - - def test_upload_picture_known_connection_error(self): - """ - This test checks if the upload_picture_known function correctly raise an exception if the connection to the db fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) - with self.assertRaises(Exception): - asyncio.run(datastore.upload_picture_known(mock_cursor, self.user_id, self.pic_encoded,self.container_client, self.seed_id, picture_set_id)) def test_upload_pictures(self): """ @@ -261,7 +140,7 @@ def test_upload_pictures(self): """ pictures = [self.pic_encoded,self.pic_encoded,self.pic_encoded] picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) - picture_ids = asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, picture_set_id, self.container_client, pictures, self.seed_name)) + picture_ids = asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, pictures, self.container_client, picture_set_id)) self.assertTrue(all([validator.is_valid_uuid(picture_id) for picture_id in picture_ids])) self.assertEqual(len(pictures), asyncio.run(datastore.azure_storage.get_image_count(self.container_client, str(picture_set_id)))) @@ -269,9 +148,10 @@ def test_upload_pictures_error_user_not_found(self): """ This test checks if the upload_picture_known function correctly raise an exception if the user given doesn't exist in db """ + pictures = [self.pic_encoded,self.pic_encoded,self.pic_encoded] picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.upload_pictures(self.cursor, uuid.uuid4(), picture_set_id, self.container_client,[self.pic_encoded,self.pic_encoded,self.pic_encoded], self.seed_name)) + asyncio.run(datastore.upload_pictures(self.cursor, uuid.uuid4(), pictures, self.container_client, picture_set_id)) def test_upload_pictures_connection_error(self): """ @@ -281,19 +161,11 @@ def test_upload_pictures_connection_error(self): mock_cursor.fetchone.side_effect = Exception("Connection error") picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) with self.assertRaises(Exception): - asyncio.run(datastore.upload_pictures(mock_cursor, self.user_id, picture_set_id, self.container_client,[self.pic_encoded,self.pic_encoded,self.pic_encoded], self.seed_name)) + asyncio.run(datastore.upload_pictures(mock_cursor, self.user_id, [self.pic_encoded,self.pic_encoded,self.pic_encoded], self.container_client, picture_set_id)) - def test_upload_pictures_error_seed_not_found(self): - """ - This test checks if the upload_picture_known function correctly raise an exception if the seed given doesn't exist in db - """ - picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) - with self.assertRaises(datastore.seed.SeedNotFoundError): - asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, picture_set_id, self.container_client,[self.pic_encoded,self.pic_encoded,self.pic_encoded], "unknown_seed")) - -class test_feedback(unittest.TestCase): +class test_picture_set(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) self.cursor = self.con.cursor() db.create_search_path(self.con, self.cursor,DB_SCHEMA) self.connection_str=BLOB_CONNECTION_STRING @@ -303,222 +175,40 @@ def setUp(self): self.image_byte_array = io.BytesIO() self.image.save(self.image_byte_array, format="TIFF") self.pic_encoded = self.image.tobytes() + #self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) self.container_name='test-container' self.user_id=datastore.User.get_id(self.user_obj) - self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,'test-user')) - base_dir = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(base_dir, 'inference_result.json') - with open(file_path) as file: - self.inference= json.load(file) - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client)) - model_id = "test_model_id" - self.registered_inference = asyncio.run(datastore.register_inference_result(self.cursor,self.user_id,self.inference, picture_id, model_id)) - self.registered_inference["userId"] = self.user_id - self.mock_box = { - "topX": 123, - "topY": 456, - "bottomX": 789, - "bottomY": 123 - } - self.inference_id = self.registered_inference.get("inferenceId") - self.boxes_id = [] - self.top_id = [] - self.unreal_seed_id= datastore.seed.new_seed(self.cursor, "unreal_seed") - for box in self.registered_inference["boxes"]: - self.boxes_id.append(box["boxId"]) - self.top_id.append(box["top_id"]) - box["classId"] = datastore.seed.get_seed_id(self.cursor, box["label"]) - + self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) + self.folder_name = "test_folder" + self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name)) + self.pictures_ids = asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, [self.pic_encoded, self.pic_encoded, self.pic_encoded], self.container_client, self.picture_set_id)) def tearDown(self): self.con.rollback() self.container_client.delete_container() db.end_query(self.con, self.cursor) - - def test_new_perfect_inference_feedback(self): - """ - This test checks if the new_perfect_inference_feeback function correctly updates the inference object after a perfect feedback is given - """ - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, self.inference_id, self.user_id, self.boxes_id)) - for i in range(len(self.boxes_id)) : - object = datastore.inference.get_inference_object(self.cursor, self.boxes_id[i]) - # verified_id must be equal to top_id - self.assertEqual(str(object[4]), self.top_id[i]) - # valid column must be true - self.assertTrue(object[5]) - - def test_new_perfect_inference_feedback_error_verified_inference(self): - """ - This test checks if the new_perfect_inference_feeback function correctly raise an exception if the inference given is already verified - """ - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, self.inference_id, self.user_id, self.boxes_id)) - self.assertTrue(datastore.inference.is_inference_verified(self.cursor, self.inference_id)) - with self.assertRaises(datastore.inference.InferenceAlreadyVerifiedError): - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, self.inference_id, self.user_id, self.boxes_id)) - - - def test_new_perfect_inference_feedback_error_inference_not_found(self): - """ - This test checks if the new_perfect_inference_feeback function correctly raise an exception if the inference given doesn't exist in db - """ - with self.assertRaises(datastore.inference.InferenceNotFoundError): - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, str(uuid.uuid4()), self.user_id, self.boxes_id)) - - def test_new_perfect_inference_feedback_error_inference_object_not_found(self): - """ - This test checks if the new_perfect_inference_feeback function correctly raise an exception if one of the inference object given doesn't exist in db - """ - with self.assertRaises(datastore.inference.InferenceObjectNotFoundError): - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, self.inference_id, self.user_id, [self.boxes_id[0], str(uuid.uuid4())])) - - def test_new_perfect_inference_feedback_error_user_not_found(self): + + def test_create_picture_set(self): """ - This test checks if the new_perfect_inference_feeback function correctly raise an exception if the user given doesn't exist in db + Test the creation of a picture set """ - with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, self.inference_id, str(uuid.uuid4()), self.boxes_id)) + picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) + self.assertTrue(validator.is_valid_uuid(picture_set_id)) - def test_new_perfect_inference_feedback_connection_error(self): + def test_create_picture_set_connection_error(self): """ - This test checks if the new_perfect_inference_feeback function correctly raise an exception if the connection to the db fails + This test checks if the create_picture_set function correctly raise an exception if the connection to the db fails """ mock_cursor = MagicMock() mock_cursor.fetchone.side_effect = Exception("Connection error") with self.assertRaises(Exception): - asyncio.run(datastore.new_perfect_inference_feeback(mock_cursor, self.inference_id, self.user_id, self.boxes_id)) - - def test_new_correction_inference_feedback(self): + asyncio.run(datastore.create_picture_set(mock_cursor, self.container_client, 0, self.user_id)) + + def test_create_picture_set_error_user_not_found(self): """ - This test checks if the new_correction_inference_feeback function correctly + This test checks if the create_picture_set function correctly raise an exception if the user given doesn't exist in db """ - self.assertTrue(validator.is_valid_uuid(self.inference_id)) - - asyncio.run(datastore.new_correction_inference_feedback(self.cursor, self.registered_inference, 1)) - for i in range(len(self.boxes_id)) : - object = datastore.inference.get_inference_object(self.cursor, self.boxes_id[i]) - # verified_id must be equal to top_id - self.assertEqual(str(object[4]), self.top_id[i]) - # valid column must be true - self.assertTrue(object[6]) - - def test_new_correction_inference_feedback_new_guess(self): - """ - This test checks if the new_correction_inference_feeback function correctly when another guess is verified - """ - self.assertTrue(validator.is_valid_uuid(self.inference_id)) - new_top_ids =[] - for box in self.registered_inference["boxes"]: - box["label"] = box["topN"][1]["label"] - box["classId"] = datastore.seed.get_seed_id(self.cursor, box["label"]) - new_top_ids.append(box["topN"][1]["object_id"]) - asyncio.run(datastore.new_correction_inference_feedback(self.cursor, self.registered_inference, 1)) - for i in range(len(self.boxes_id)) : - object_db = datastore.inference.get_inference_object(self.cursor, self.boxes_id[i]) - # verified_id must be equal to top_id - self.assertTrue(str(object_db[4]) == new_top_ids[i]) - # valid column must be true - self.assertTrue(object_db[6]) - - def test_new_correction_inference_feedback_box_edited(self): - """ - This test checks if the new_correction_inference_feeback function correctly when the box metadata is updated - """ - self.assertTrue(validator.is_valid_uuid(self.inference_id)) - for box in self.registered_inference["boxes"]: - box["box"]= self.mock_box - asyncio.run(datastore.new_correction_inference_feedback(self.cursor, self.registered_inference, 1)) - for box in self.registered_inference["boxes"] : - object_db = datastore.inference.get_inference_object(self.cursor, box["boxId"]) - # The new box metadata must be updated - self.assertDictEqual(object_db[1], self.mock_box) - # The top_id must be equal to the previous top_id - self.assertEqual(str(object_db[4]), box["top_id"]) - # valid column must be true - self.assertTrue(object_db[6]) - - def test_new_correction_inference_feedback_not_guess(self): - """ - This test checks if the new_correction_inference_feeback function correctly when the box is not a guess - """ - self.assertTrue(validator.is_valid_uuid(self.inference_id)) - for box in self.registered_inference["boxes"]: - box["label"] = "unreal_seed" - box["classId"] = self.unreal_seed_id - asyncio.run(datastore.new_correction_inference_feedback(self.cursor, self.registered_inference, 1)) - for i in range(len(self.boxes_id)) : - object_db = datastore.inference.get_inference_object(self.cursor, self.boxes_id[i]) - # verified_id must be equal to the new_top_id - new_top_id = datastore.inference.get_seed_object_id(self.cursor,self.unreal_seed_id,object_db[0]) - self.assertTrue(validator.is_valid_uuid(new_top_id)) - self.assertEqual(str(object_db[4]),str(new_top_id)) - # valid column must be true - self.assertTrue(object_db[6]) - - def test_new_correction_inference_feedback_not_valid(self): - """ - This test checks if the new_correction_inference_feeback function correctly when the box is not a guess - """ - self.assertTrue(validator.is_valid_uuid(self.inference_id)) - for box in self.registered_inference["boxes"]: - box["label"] = "" - box["classId"] = "" - asyncio.run(datastore.new_correction_inference_feedback(self.cursor, self.registered_inference, 1)) - for i in range(len(self.boxes_id)) : - object_db = datastore.inference.get_inference_object(self.cursor, self.boxes_id[i]) - # verified_id must not be an id - self.assertEqual(object_db[4], None) - # valid column must be false - self.assertFalse(object_db[6]) - - def test_new_correction_inference_feedback_unknown_seed(self): - """ - This test checks if the new_correction_inference_feeback function correctly when the box is not a guess - """ - self.assertTrue(validator.is_valid_uuid(self.inference_id)) - for box in self.registered_inference["boxes"]: - box["label"] = "unknown_seed" - box["classId"] = "" - asyncio.run(datastore.new_correction_inference_feedback(self.cursor, self.registered_inference, 1)) - for i in range(len(self.boxes_id)) : - object_db = datastore.inference.get_inference_object(self.cursor, self.boxes_id[i]) - # verified_id must be equal to an id - self.assertTrue(validator.is_valid_uuid(str(object_db[4]))) - # valid column must be true - self.assertTrue(object_db[6]) - -class test_picture_set(unittest.TestCase): - def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) - self.cursor = self.con.cursor() - db.create_search_path(self.con, self.cursor,DB_SCHEMA) - self.connection_str=BLOB_CONNECTION_STRING - self.user_email="test@email" - self.user_obj= asyncio.run(datastore.new_user(self.cursor,self.user_email,self.connection_str,'test-user')) - self.image = Image.new("RGB", (1980, 1080), "blue") - self.image_byte_array = io.BytesIO() - self.image.save(self.image_byte_array, format="TIFF") - self.pic_encoded = self.image.tobytes() - #self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) - self.container_name='test-container' - self.user_id=datastore.User.get_id(self.user_obj) - self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,'test-user')) - self.seed_name = "test-name" - self.seed_id = seed_query.new_seed(self.cursor, self.seed_name) - self.folder_name = "test_folder" - self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name)) - self.pictures_id =[] - for i in range(3) : - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) - self.pictures_id.append(picture_id) - with open("tests/inference_result.json") as file: - self.inference= json.load(file) - - self.dev_user_id=datastore.user.get_user_id(self.cursor, os.environ["DEV_USER_EMAIL"]) - self.dev_container_client = asyncio.run(datastore.get_user_container_client(self.dev_user_id)) - - def tearDown(self): - self.con.rollback() - self.container_client.delete_container() - db.end_query(self.con, self.cursor) + with self.assertRaises(datastore.user.UserNotFoundError): + asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, uuid.uuid4())) def test_get_picture_sets_info(self) : """ @@ -552,73 +242,7 @@ def test_get_picture_sets_info_error_connection_error(self): mock_cursor.fetchone.side_effect = Exception("Connection error") with self.assertRaises(Exception): asyncio.run(datastore.get_picture_sets_info(mock_cursor, self.user_id)) - - - def test_find_validated_pictures(self): - """ - This test checks if the find_validated_pictures function correctly returns the validated pictures of a picture_set - """ - - self.assertEqual(len(asyncio.run(datastore.find_validated_pictures(self.cursor, str(self.user_id), str(self.picture_set_id)))), 0, "No validated pictures should be found") - - inferences = [] - for picture_id in self.pictures_id : - # Using deepcopy to ensure each inference is a unique object without shared references - inference_copy = deepcopy(self.inference) - inference = asyncio.run(datastore.register_inference_result(self.cursor,self.user_id,inference_copy, picture_id, "test_model_id")) - inferences.append(inference) - - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, inferences[0]["inferenceId"], self.user_id, [box["boxId"] for box in inferences[0]["boxes"]])) - - self.assertEqual(len(asyncio.run(datastore.find_validated_pictures(self.cursor, str(self.user_id), str(self.picture_set_id)))), 1, "One validated pictures should be found") - - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, inferences[1]["inferenceId"], self.user_id, [box["boxId"] for box in inferences[1]["boxes"]])) - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, inferences[2]["inferenceId"], self.user_id, [box["boxId"] for box in inferences[2]["boxes"]])) - - self.assertEqual(len(asyncio.run(datastore.find_validated_pictures(self.cursor, str(self.user_id), str(self.picture_set_id)))), 3, "One validated pictures should be found") - - - def test_find_validated_pictures_error_user_not_found(self): - """ - This test checks if the find_validated_pictures function correctly raise an exception if the user given doesn't exist in db - """ - pictures_id =[] - for i in range(3) : - picture_id = asyncio.run(datastore.upload_picture_unknown(self.cursor, self.user_id, self.pic_encoded,self.container_client, self.picture_set_id)) - pictures_id.append(picture_id) - - with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.find_validated_pictures(self.cursor, str(uuid.uuid4()), str(self.picture_set_id))) - - def test_find_validated_pictures_error_connection_error(self): - """ - This test checks if the find_validated_pictures function correctly raise an exception if the connection to the db fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - asyncio.run(datastore.find_validated_pictures(mock_cursor, str(self.user_id), str(self.picture_set_id))) - - def test_find_validated_pictures_error_picture_set_not_found(self): - """ - This test checks if the find_validated_pictures function correctly raise an exception if the picture set given doesn't exist in db - """ - with self.assertRaises(datastore.picture.PictureSetNotFoundError): - asyncio.run(datastore.find_validated_pictures(self.cursor, str(self.user_id), str(uuid.uuid4()))) - - def test_find_validated_pictures_error_not_owner(self): - """ - This test checks if the find_validated_pictures function correctly raise an exception if the user is not the owner of the picture set - """ - not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) - not_owner_user_id=datastore.User.get_id(not_owner_user_obj) - - with self.assertRaises(datastore.UserNotOwnerError): - asyncio.run(datastore.find_validated_pictures(self.cursor, str(not_owner_user_id), str(self.picture_set_id))) - - container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,'test-user')) - container_client.delete_container() - + def test_delete_picture_set_permanently(self): """ This test checks the delete_picture_set_permanently function @@ -663,7 +287,7 @@ def test_delete_picture_set_permanently_error_not_owner(self): with self.assertRaises(datastore.UserNotOwnerError): asyncio.run(datastore.delete_picture_set_permanently(self.cursor, str(not_owner_user_id), str(self.picture_set_id), self.container_client)) - container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,'test-user')) + container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) container_client.delete_container() def test_delete_picture_set_permanently_error_default_folder(self): @@ -673,126 +297,6 @@ def test_delete_picture_set_permanently_error_default_folder(self): general_folder_id = datastore.user.get_default_picture_set(self.cursor, self.user_id) with self.assertRaises(datastore.picture.PictureSetDeleteError): asyncio.run(datastore.delete_picture_set_permanently(self.cursor, str(self.user_id), str(general_folder_id), self.container_client)) - - def test_delete_picture_set_with_archive(self): - """ - This test checks if the delete_picture_set_with_archive function correctly archive the picture set in dev container and delete it from user container - """ - # Create inferences for pictures in the picture set - inferences = [] - for picture_id in self.pictures_id : - # Using deepcopy to ensure each inference is a unique object without shared references - inference_copy = deepcopy(self.inference) - inference = asyncio.run(datastore.register_inference_result(self.cursor,self.user_id,inference_copy, picture_id, "test_model_id")) - inferences.append(inference) - # Validate 2 of 3 pictures in the picture set - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, inferences[1]["inferenceId"], self.user_id, [box["boxId"] for box in inferences[1]["boxes"]])) - asyncio.run(datastore.new_perfect_inference_feeback(self.cursor, inferences[2]["inferenceId"], self.user_id, [box["boxId"] for box in inferences[2]["boxes"]])) - validated_pictures = asyncio.run(datastore.find_validated_pictures(self.cursor, str(self.user_id), str(self.picture_set_id))) - - dev_nb_folders = len(asyncio.run(datastore.get_picture_sets_info(self.cursor, self.dev_user_id))) - # Check there is the right number of picture sets in db for each user - self.assertEqual(len(asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id))), 2) - - dev_picture_set_id = asyncio.run(datastore.delete_picture_set_with_archive(self.cursor, str(self.user_id), str(self.picture_set_id), self.container_client)) - - # Check there is the right number of picture sets in db for each user after moving - self.assertEqual(len(asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id))), 1) - self.assertEqual(len(asyncio.run(datastore.get_picture_sets_info(self.cursor, self.dev_user_id))), dev_nb_folders + 1) - - # Check blobs have also moved in blob storage - for picture_id in validated_pictures : - blob_name = "{}/{}.png".format(self.folder_name, picture_id) - with self.assertRaises(Exception): - asyncio.run(datastore.azure_storage.get_blob(self.container_client, blob_name)) - - blob_name = "{}/{}/{}.png".format(self.user_id, self.folder_name, str(picture_id)) - blob = asyncio.run(datastore.azure_storage.get_blob(self.dev_container_client, blob_name)) - self.assertEqual(blob, self.pic_encoded) - - # TEAR DOWN - # Delete the user folder in the blob storage - asyncio.run(datastore.azure_storage.delete_folder(self.dev_container_client, str(dev_picture_set_id))) - asyncio.run(datastore.azure_storage.delete_folder(self.dev_container_client, str(self.user_id))) - - def test_delete_picture_set_with_archive_error_user_not_found(self): - """ - This test checks if the delete_picture_set_with_archive function correctly raise an exception if the user given doesn't exist in db - """ - with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.delete_picture_set_with_archive(self.cursor, str(uuid.uuid4()), str(self.picture_set_id), self.container_client)) - - def test_delete_picture_set_with_archive_error_connection_error(self): - """ - This test checks if the delete_picture_set_with_archive function correctly raise an exception if the connection to the db fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - asyncio.run(datastore.delete_picture_set_with_archive(mock_cursor, str(self.user_id), str(self.picture_set_id), self.container_client)) - - def test_delete_picture_set_with_archive_error_picture_set_not_found(self): - """ - This test checks if the delete_picture_set_with_archive function correctly raise an exception if the picture set given doesn't exist in db - """ - with self.assertRaises(datastore.picture.PictureSetNotFoundError): - asyncio.run(datastore.delete_picture_set_with_archive(self.cursor, str(self.user_id), str(uuid.uuid4()), self.container_client)) - - def test_delete_picture_set_with_archive_error_not_owner(self): - """ - This test checks if the delete_picture_set_with_archive function correctly raise an exception if the user is not the owner of the picture set - """ - not_owner_user_obj= asyncio.run(datastore.new_user(self.cursor,"notowner@email",self.connection_str,'test-user')) - not_owner_user_id=datastore.User.get_id(not_owner_user_obj) - - with self.assertRaises(datastore.UserNotOwnerError): - asyncio.run(datastore.delete_picture_set_with_archive(self.cursor, str(not_owner_user_id), str(self.picture_set_id), self.container_client)) - - container_client = asyncio.run(datastore.get_user_container_client(not_owner_user_id,'test-user')) - container_client.delete_container() - - def test_delete_picture_set_with_archive_error_default_folder(self): - """ - This test checks if the delete_picture_set_with_archive function correctly raise an exception if the user want to delete the folder "General" - """ - general_folder_id = datastore.user.get_default_picture_set(self.cursor, self.user_id) - with self.assertRaises(datastore.picture.PictureSetDeleteError): - asyncio.run(datastore.delete_picture_set_with_archive(self.cursor, str(self.user_id), str(general_folder_id), self.container_client)) - -class test_analysis(unittest.TestCase): - def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) - self.cursor = self.con.cursor() - db.create_search_path(self.con, self.cursor,DB_SCHEMA) - self.connection_str=BLOB_CONNECTION_STRING - - self.image = Image.new("RGB", (1980, 1080), "blue") - self.image_byte_array = io.BytesIO() - self.image.save(self.image_byte_array, format="TIFF") - self.pic_encoded = self.image.tobytes() - - self.container_name='dev' - # self.credentials = datastore.blob.get_account_sas(os.environ["FERTISCAN_BLOB_ACCOUNT"], os.environ["FERTISCAN_BLOB_KEY"]) - # self.container_client = asyncio.run(datastore.azure_storage.mount_container(self.connection_str,self.container_name,True,'test',self.credentials)) - self.container_client = 'on hold' - self.analysis_dict = { - "company_name": "GreenGrow Fertilizers Inc.", - "company_address": "123 Greenway Blvd Springfield IL 62701 USA", - "company_website": "www.greengrowfertilizers.com", - } - - def tearDown(self): - self.con.rollback() - # self.container_client.delete_container() - db.end_query(self.con, self.cursor) - - def test_registere_analysis(self): - """ - Test the register analysis function - """ - # self.assertTrue(self.container_client.exists()) - analysis = asyncio.run(datastore.register_analysis(self.cursor, self.container_client,self.analysis_dict,self.pic_encoded,"","General")) - self.assertTrue(validator.is_valid_uuid(analysis["analysis_id"])) if __name__ == "__main__": unittest.main() diff --git a/tests/test_db.py b/tests/test_db.py index 601ea9a1..fad0de34 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -10,36 +10,17 @@ from PIL import Image import io import base64 -from time import sleep from unittest.mock import MagicMock import datastore.db.__init__ as db from datastore.db.metadata import picture as picture_data from datastore.db.metadata import picture_set as picture_set_data from datastore.db.metadata import validator -from datastore.db.queries import inference, picture, seed, user +from datastore.db.queries import picture, user DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": - raise ValueError("NACHET_DB_URL is not set") - -DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") -if DB_SCHEMA is None or DB_SCHEMA == "": - raise ValueError("NACHET_SCHEMA_TESTING is not set") - - -DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") -if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": - raise ValueError("NACHET_DB_URL is not set") - -DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") -if DB_SCHEMA is None or DB_SCHEMA == "": - raise ValueError("NACHET_SCHEMA_TESTING is not set") - - -DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") -if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": - raise ValueError("NACHET_DB_URL is not set") + raise ValueError("NACHET_DB_URL_TESTING is not set") DB_SCHEMA = os.environ.get("NACHET_SCHEMA_TESTING") if DB_SCHEMA is None or DB_SCHEMA == "": @@ -49,10 +30,9 @@ # -------------------- USER FUNCTIONS -------------------- class test_user_functions(unittest.TestCase): def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) self.cursor = self.con.cursor() - db.create_search_path(self.con, self.cursor,DB_SCHEMA) + db.create_search_path(self.con, self.cursor, DB_SCHEMA) self.email = "test@email.gouv.ca" def tearDown(self): @@ -232,121 +212,13 @@ def test_get_container_url_error(self): user.get_container_url(mock_cursor, user_id) -# -------------------- SEED FUNCTIONS -------------------- -class test_seed_functions(unittest.TestCase): - def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING) - self.con = db.connect_db(DB_CONNECTION_STRING) - self.cursor = db.cursor(self.con) - db.create_search_path(self.con, self.cursor,DB_SCHEMA) - self.seed_name = "test-name" - - def tearDown(self): - self.con.rollback() - db.end_query(self.con, self.cursor) - - def test_get_all_seeds_names(self): - """ - This test checks if the get_all_seeds_names function returns a list of seeds - with at least the one seed we added - """ - seed.new_seed(self.cursor, self.seed_name) - seeds = seed.get_all_seeds_names(self.cursor) - - self.assertNotEqual(len(seeds), 0) - self.assertIn((self.seed_name,), seeds) - - def test_get_all_seeds_names_error(self): - """ - This test checks if the get_all_seeds_names function raises an exception when the connection fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchall.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - seed.get_all_seeds_names(mock_cursor) - - def test_get_seed_id(self): - """ - This test checks if the get_seed_id function returns the correct UUID - """ - seed_uuid = seed.new_seed(self.cursor, self.seed_name) - fetch_id = seed.get_seed_id(self.cursor, self.seed_name) - - self.assertTrue(validator.is_valid_uuid(fetch_id)) - self.assertEqual(seed_uuid, fetch_id) - - def test_get_nonexistant_seed_id(self): - """ - This test checks if the get_seed_id function raises an exception when the seed does not exist - """ - with self.assertRaises(seed.SeedNotFoundError): - seed.get_seed_id(self.cursor, "nonexistant_seed") - - def test_get_seed_id_error(self): - """ - This test checks if the get_seed_id function raises an exception when the connection fails - """ - seed.new_seed(self.cursor, self.seed_name) - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - seed.get_seed_id(mock_cursor, self.seed_name) - - def test_new_seed(self): - """ - This test checks if the new_seed function returns a valid UUID - """ - seed_id = seed.new_seed(self.cursor, self.seed_name) - - self.assertTrue(validator.is_valid_uuid(seed_id)) - - def test_new_seed_error(self): - """ - This test checks if the new_seed function raises an exception when the connection fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(seed.SeedCreationError): - seed.new_seed(mock_cursor, self.seed_name) - - def test_is_seed_registered(self): - """ - This test checks if the is_seed_registered function returns the correct value - for a seed that is not yet registered and one that is. - """ - self.assertFalse( - seed.is_seed_registered(self.cursor, self.seed_name), - "The seed should not already be registered", - ) - - seed.new_seed(self.cursor, self.seed_name) - - self.assertTrue( - seed.is_seed_registered(self.cursor, self.seed_name), - "The seed should be registered", - ) - - def test_is_seed_registered_error(self): - """ - This test checks if the is_seed_registered function raises an exception when the connection fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(Exception): - seed.is_seed_registered(mock_cursor, self.seed_name) - - # -------------------- PICTURE FUNCTIONS -------------------- class test_pictures_functions(unittest.TestCase): def setUp(self): # prepare the connection and cursor - self.con = db.connect_db(DB_CONNECTION_STRING) - self.con = db.connect_db(DB_CONNECTION_STRING) + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) self.cursor = db.cursor(self.con) - - # prepare the seed - self.seed_name = "test seed" - self.seed_id = seed.new_seed(self.cursor, self.seed_name) + db.create_search_path(self.con, self.cursor, DB_SCHEMA) # prepare the user self.user_id = user.register_user(self.cursor, "test@email") @@ -416,35 +288,6 @@ def test_new_picture_set_error(self): with self.assertRaises(picture.PictureSetCreationError): picture.new_picture_set(mock_cursor, self.picture_set, self.user_id) - def test_new_picture(self): - """ - This test checks if the new_picture function returns a valid UUID - """ - # prepare the picture_set - picture_set_id = picture.new_picture_set( - self.cursor, self.picture_set, self.user_id - ) - - # create the new picture in the db - picture_id = picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed - ) - - self.assertTrue( - validator.is_valid_uuid(picture_id), "The picture_id is not a valid UUID" - ) - - def test_new_picture_error(self): - """ - This test checks if the new_picture function raises an exception when the connection fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(picture.PictureUploadError): - picture.new_picture( - mock_cursor, self.picture, str(uuid.uuid4()), self.seed_id, self.nb_seed - ) - def test_get_inexisting_picture_set(self): """ This test checks if the get_picture_set function raises an exception when the picture_set does not exist @@ -483,8 +326,8 @@ def test_get_picture(self): ) # prepare picture - picture_id = picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + picture_id = picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) # test the function @@ -503,9 +346,10 @@ def test_update_picture_metadata(self): ) # prepare picture - picture_id = picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + picture_id = picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) + new_picture = picture_data.build_picture( self.pic_encoded, "www.link.com", 6, 1.0, "" ) @@ -515,6 +359,7 @@ def test_update_picture_metadata(self): new_picture = json.loads(new_picture) # get the updated metadata new_picture_metadata = picture.get_picture(self.cursor, picture_id) + self.assertEqual( new_picture_metadata["user_data"], new_picture["user_data"], @@ -659,11 +504,12 @@ def test_count_pictures(self): picture_set_id = picture.new_picture_set( self.cursor, self.picture_set, self.user_id, self.folder_name ) - picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + + picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) - picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) count = picture.count_pictures(self.cursor, picture_set_id) @@ -676,8 +522,9 @@ def test_count_pictures_error(self): picture_set_id = picture.new_picture_set( self.cursor, self.picture_set, self.user_id, self.folder_name ) - picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + + picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) mock_cursor = MagicMock() @@ -700,11 +547,11 @@ def test_get_picture_set_pictures(self): "The number of pictures is not the expected one", ) - pictureId = picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + pictureId1 = picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) - pictureId2 = picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + pictureId2 = picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) pictures = picture.get_picture_set_pictures(self.cursor, picture_set_id) @@ -712,7 +559,7 @@ def test_get_picture_set_pictures(self): len(pictures), 2, "The number of pictures is not the expected one" ) self.assertEqual( - pictures[0][0], pictureId, "The first picture is not the expected one" + pictures[0][0], pictureId1, "The first picture is not the expected one" ) self.assertEqual( pictures[1][0], pictureId2, "The second picture is not the expected one" @@ -725,8 +572,9 @@ def test_get_picture_set_pictures_error(self): picture_set_id = picture.new_picture_set( self.cursor, self.picture_set, self.user_id, self.folder_name ) - picture.new_picture( - self.cursor, self.picture, picture_set_id, self.seed_id, self.nb_seed + + picture.new_picture_unknown( + self.cursor, self.picture, picture_set_id, self.nb_seed ) mock_cursor = MagicMock() @@ -739,490 +587,88 @@ def test_change_picture_set_id(self): old_picture_set_id = picture.new_picture_set( self.cursor, self.picture_set, self.user_id, self.folder_name ) - picture.new_picture( - self.cursor, self.picture, old_picture_set_id, self.seed_id, self.nb_seed - ) - picture.new_picture( - self.cursor, self.picture, old_picture_set_id, self.seed_id, self.nb_seed - ) - new_picture_set_id = picture.new_picture_set( - self.cursor, self.picture_set, self.user_id, self.folder_name - ) - old_picture_set = picture.get_picture_set_pictures(self.cursor, old_picture_set_id) - self.assertEqual(len(old_picture_set), 2, "The number of pictures is not the expected one") - new_picture_set = picture.get_picture_set_pictures(self.cursor, new_picture_set_id) - self.assertEqual(len(new_picture_set), 0, "The number of pictures is not the expected one") - - picture.change_picture_set_id(self.cursor, str(self.user_id), str(old_picture_set_id), str(new_picture_set_id)) - old_picture_set = picture.get_picture_set_pictures(self.cursor, old_picture_set_id) - self.assertEqual(len(old_picture_set), 0, "The number of pictures is not the expected one") - new_picture_set = picture.get_picture_set_pictures(self.cursor, new_picture_set_id) - self.assertEqual(len(new_picture_set), 2, "The number of pictures is not the expected one") - - def test_change_picture_set_id_error(self): - old_picture_set_id = picture.new_picture_set( - self.cursor, self.picture_set, self.user_id, self.folder_name + picture.new_picture_unknown( + self.cursor, self.picture, old_picture_set_id, self.nb_seed ) - picture.new_picture( - self.cursor, self.picture, old_picture_set_id, self.seed_id, self.nb_seed + picture.new_picture_unknown( + self.cursor, self.picture, old_picture_set_id, self.nb_seed ) - new_picture_set_id = picture.new_picture_set( - self.cursor, self.picture_set, self.user_id, self.folder_name - ) - - mock_cursor = MagicMock() - mock_cursor.execute.side_effect = Exception("Connection error") - - with self.assertRaises(picture.PictureUpdateError): - picture.change_picture_set_id( - mock_cursor, self.user_id, old_picture_set_id, new_picture_set_id - ) - def test_change_picture_set_id_user_error(self): - old_picture_set_id = picture.new_picture_set( - self.cursor, self.picture_set, self.user_id, self.folder_name - ) - picture.new_picture( - self.cursor, self.picture, old_picture_set_id, self.seed_id, self.nb_seed - ) new_picture_set_id = picture.new_picture_set( self.cursor, self.picture_set, self.user_id, self.folder_name ) - - with self.assertRaises(picture.PictureUpdateError): - picture.change_picture_set_id( - self.cursor, str(uuid.uuid4()), old_picture_set_id, new_picture_set_id - ) - - -# -------------------- INFERENCE FUNCTIONS -------------------- -class test_inference_functions(unittest.TestCase): - def setUp(self): - # prepare the connection and cursor - self.con = db.connect_db(DB_CONNECTION_STRING) - self.con = db.connect_db(DB_CONNECTION_STRING) - self.cursor = db.cursor(self.con) - - # prepare the seed - self.seed_name = "test seed" - self.seed_id = seed.new_seed(self.cursor, self.seed_name) - - # prepare the user - self.user_id = user.register_user(self.cursor, "test@email") - - # prepare the picture_set and picture - self.image = Image.new("RGB", (1980, 1080), "blue") - self.image_byte_array = io.BytesIO() - self.image.save(self.image_byte_array, format="TIFF") - self.pic_encoded = base64.b64encode(self.image_byte_array.getvalue()).decode( - "utf8" - ) - self.picture_set = picture_set_data.build_picture_set(self.user_id, 1) - self.nb_seed = 1 - self.picture = picture_data.build_picture( - self.pic_encoded, "www.link.com", self.nb_seed, 1.0, "" - ) - self.picture_set_id = picture.new_picture_set( - self.cursor, self.picture_set, self.user_id - ) - self.picture_id = picture.new_picture( - self.cursor, self.picture, self.picture_set_id, self.seed_id, self.nb_seed - ) - with open("tests/inference_example.json", "r") as f: - self.inference = json.loads(f.read()) - self.inference_trim = ( - '{"filename": "inference_example", "totalBoxes": 1, "totalBoxes": 1}' + old_picture_set = picture.get_picture_set_pictures( + self.cursor, old_picture_set_id ) - self.type = 1 - - def tearDown(self): - self.con.rollback() - db.end_query(self.con, self.cursor) - - def test_new_inference(self): - """ - This test checks if the new_inference function returns a valid UUID - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - - self.assertTrue( - validator.is_valid_uuid(inference_id), - "The inference_id is not a valid UUID", - ) - - def test_new_inference_error(self): - """ - This test checks if the new_inference function raises an exception when the connection fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(inference.InferenceCreationError): - inference.new_inference( - mock_cursor, - self.inference_trim, - self.user_id, - self.picture_id, - self.type, - ) - - def test_new_inference_obj(self): - """ - This test checks if the new_inference_object function returns a valid UUID - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - for box in self.inference["boxes"]: - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(box), self.type - ) - self.assertTrue( - validator.is_valid_uuid(inference_obj_id), - "The inference_obj_id is not a valid UUID", - ) - - def test_new_inference_obj_error(self): - """ - This test checks if the new_inference_object function raises an exception when the connection fails - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - - with self.assertRaises(inference.InferenceCreationError): - inference.new_inference_object( - mock_cursor, inference_id, self.inference["boxes"][0], self.type - ) - - def test_new_seed_object(self): - """ - This test checks if the new_seed_object function returns a valid UUID - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - for box in self.inference["boxes"]: - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(box), self.type - ) - seed_obj_id = inference.new_seed_object( - self.cursor, self.seed_id, inference_obj_id, box["score"] - ) - self.assertTrue( - validator.is_valid_uuid(seed_obj_id), - "The seed_obj_id is not a valid UUID", - ) - - def test_new_seed_object_error(self): - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type - ) - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - with self.assertRaises(inference.SeedObjectCreationError): - inference.new_seed_object(mock_cursor, self.seed_id, inference_obj_id, 32.1) - - def test_get_inference(self): - """ - This test checks if the get_inference function returns a correctly build inference - TODO : Add test for not existing inference - """ - inference_trim = json.loads(self.inference_trim) - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_data = inference.get_inference(self.cursor, str(inference_id)) - # inference_json=json.loads(inference_data) - self.assertGreaterEqual( - len(inference_trim), - len(inference_data), - "The inference is not correctly build and has more keys than expected", - ) - for key in inference_trim: - self.assertTrue( - key in inference_data, f"The key: {key} is not in the inference" - ) - self.assertEqual( - inference_trim[key], - inference_data[key], - f"The value ({inference_data[key]}) of the key: {key} is not the same as the expected one: {inference_trim[key]}", - ) - - def test_get_inference_object(self): - """ - This test checks if the get_inference_object function returns a correctly build object - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type - ) - - inference_obj = inference.get_inference_object( - self.cursor, str(inference_obj_id) - ) - - self.assertEqual( - len(inference_obj), - 10, - "The inference object hasn't the number of keys expected", - ) - self.assertEqual( - inference_obj[0], - inference_obj_id, - "The inference object id is not the same as the expected one", - ) - - def test_get_inference_object_error(self): - """ - This test checks if the get_inference_object function raise an error if the inference oject does not exist - """ - inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = "00000000-0000-0000-0000-000000000000" - - with self.assertRaises(Exception): - inference.get_inference_object(self.cursor, str(inference_obj_id)) - - def test_get_objects_by_inference(self): - """ - This test checks if the get_objects_by_inference function returns the corrects objects for an inference - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - total_boxes = len(self.inference["boxes"]) - objects_id = [] - for box in self.inference["boxes"]: - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(box), self.type - ) - objects_id.append(inference_obj_id) - - objects = inference.get_objects_by_inference(self.cursor, inference_id) self.assertEqual( - len(objects), - total_boxes, - "The number of objects is not the same as the expected one", - ) - for object in objects: - self.assertEqual( - object[2], - inference_id, - "The inference id is not the same as the expected one", - ) - self.assertTrue( - object[0] in objects_id, - "The object id is not in the list of expected objects", - ) - - def test_get_inference_object_top_id(self): - """ - This test checks if the get_inference_object_top_id function returns the correct top_id of an inference object - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + len(old_picture_set), 2, "The number of pictures is not the expected one" ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type + new_picture_set = picture.get_picture_set_pictures( + self.cursor, new_picture_set_id ) - seed_obj_id = inference.new_seed_object( - self.cursor, - self.seed_id, - inference_obj_id, - self.inference["boxes"][0]["score"], - ) - - inference.set_inference_object_top_id( - self.cursor, inference_obj_id, seed_obj_id - ) - top_id = inference.get_inference_object_top_id(self.cursor, inference_obj_id) - self.assertEqual( - seed_obj_id, top_id, "The verified_id is not the same as the expected one" + len(new_picture_set), 0, "The number of pictures is not the expected one" ) - def test_set_inference_object_verified_id(self): - """ - This test checks if the set_inference_object_verified_id function returns a correctly update inference object - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type - ) - previous_inference_obj = inference.get_inference_object( - self.cursor, inference_obj_id - ) - seed_obj_id = inference.new_seed_object( + picture.change_picture_set_id( self.cursor, - self.seed_id, - inference_obj_id, - self.inference["boxes"][0]["score"], + str(self.user_id), + str(old_picture_set_id), + str(new_picture_set_id), ) - # Sleep to see a difference in the updated_at date of the object - sleep(3) - inference.set_inference_object_verified_id( - self.cursor, inference_obj_id, seed_obj_id + old_picture_set = picture.get_picture_set_pictures( + self.cursor, old_picture_set_id ) - inference_obj = inference.get_inference_object(self.cursor, inference_obj_id) self.assertEqual( - str(inference_obj[4]), - str(seed_obj_id), - "The verified_id is not the same as the expected one", + len(old_picture_set), 0, "The number of pictures is not the expected one" ) - # this test is not working because the trigger to update the update_at field is missing - self.assertNotEqual( - inference_obj[8], - previous_inference_obj[8], - "The update_at field is not updated", + new_picture_set = picture.get_picture_set_pictures( + self.cursor, new_picture_set_id ) - - def test_set_inference_object_valid(self): - """ - This test checks if the set_inference_object_verified_id function returns a correctly update inference object - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type - ) - previous_inference_obj = inference.get_inference_object( - self.cursor, inference_obj_id - ) - # Sleep to see a difference in the updated_at date of the object - sleep(3) - - inference.set_inference_object_valid(self.cursor, inference_obj_id, True) - inference_obj = inference.get_inference_object(self.cursor, inference_obj_id) - self.assertTrue( - str(inference_obj[5]), - "The object validity is not the same as the expected one", - ) - self.assertNotEqual( - inference_obj[8], - previous_inference_obj[8], - "The update_at field is not updated", + self.assertEqual( + len(new_picture_set), 2, "The number of pictures is not the expected one" ) - inference.set_inference_object_valid(self.cursor, inference_obj_id, False) - inference_obj = inference.get_inference_object(self.cursor, inference_obj_id) - self.assertFalse( - str(inference_obj[5]), - "The object validity is not the same as the expected one", - ) - # this test is not working because the trigger to update the update_at field is missing - self.assertNotEqual( - inference_obj[8], - previous_inference_obj[8], - "The update_at field is not updated", + def test_change_picture_set_id_error(self): + old_picture_set_id = picture.new_picture_set( + self.cursor, self.picture_set, self.user_id, self.folder_name ) - def test_is_inference_verified(self): - """ - Test if is_inference_verified function correctly returns the inference status - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + picture.new_picture_unknown( + self.cursor, self.picture, old_picture_set_id, self.nb_seed ) - verified = inference.is_inference_verified(self.cursor, inference_id) - self.assertFalse(verified, "The inference verified field should be False") - - inference.set_inference_verified(self.cursor, inference_id, True) - - verified = inference.is_inference_verified(self.cursor, inference_id) - self.assertTrue(verified, "The inference should be fully verified") - - def test_verify_inference_status(self): - """ - Test if verify_inference_status function correctly updates the inference status - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type - ) - seed_obj_id = inference.new_seed_object( - self.cursor, - self.seed_id, - inference_obj_id, - self.inference["boxes"][0]["score"], + new_picture_set_id = picture.new_picture_set( + self.cursor, self.picture_set, self.user_id, self.folder_name ) - inference.verify_inference_status(self.cursor, inference_id, self.user_id) + mock_cursor = MagicMock() + mock_cursor.execute.side_effect = Exception("Connection error") - verified = inference.is_inference_verified(self.cursor, inference_id) - self.assertFalse(verified, "The inference verified field should be False") + with self.assertRaises(picture.PictureUpdateError): + picture.change_picture_set_id( + mock_cursor, self.user_id, old_picture_set_id, new_picture_set_id + ) - inference.set_inference_object_valid(self.cursor, inference_obj_id, True) - inference.set_inference_object_verified_id( - self.cursor, inference_obj_id, seed_obj_id + def test_change_picture_set_id_user_error(self): + old_picture_set_id = picture.new_picture_set( + self.cursor, self.picture_set, self.user_id, self.folder_name ) - inference.verify_inference_status(self.cursor, inference_id, self.user_id) - verified = inference.is_inference_verified(self.cursor, inference_id) - self.assertTrue(verified, "The inference should be fully verified") - - def test_get_seed_object_id(self): - """ - Test if get_seed_object_id function correctly returns the seed object id - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type - ) - seed_obj_id = inference.new_seed_object( - self.cursor, - self.seed_id, - inference_obj_id, - self.inference["boxes"][0]["score"], + picture.new_picture_unknown( + self.cursor, self.picture, old_picture_set_id, self.nb_seed ) - fetched_seed_obj_id = inference.get_seed_object_id( - self.cursor, self.seed_id, inference_obj_id - ) - self.assertEqual( - seed_obj_id, - fetched_seed_obj_id, - "The fetched seed object id is not the same as the expected one", + new_picture_set_id = picture.new_picture_set( + self.cursor, self.picture_set, self.user_id, self.folder_name ) - def test_get_not_seed_object_id(self): - """ - Test if get_seed_object_id function correctly returns the seed object id - """ - inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type - ) - inference_obj_id = inference.new_inference_object( - self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type - ) - inference.new_seed_object( - self.cursor, - self.seed_id, - inference_obj_id, - self.inference["boxes"][0]["score"], - ) - mock_seed_id = str(uuid.uuid4()) - fetched_seed_obj_id = inference.get_seed_object_id(self.cursor,mock_seed_id, inference_obj_id) - self.assertTrue(fetched_seed_obj_id is None, "The fetched seed object id should be None") + with self.assertRaises(picture.PictureUpdateError): + picture.change_picture_set_id( + self.cursor, str(uuid.uuid4()), old_picture_set_id, new_picture_set_id + ) if __name__ == "__main__": diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 5ba690f8..55b8860d 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2,8 +2,6 @@ import uuid import datastore.db.metadata.picture_set as picture_set_data import datastore.db.metadata.picture as picture_data -import datastore.db.metadata.inference as inference -import datastore.db.metadata.machine_learning as ml_data import io from PIL import Image from datetime import date @@ -99,169 +97,6 @@ def test_build_picture_set(self): self.assertEqual(data["audit_trail"]["access_log"], "picture_set accessed") self.assertEqual(data["audit_trail"]["privacy_flag"], False) -class test_inference_functions(unittest.TestCase): - def setUp(self): - self.filename="test.jpg" - self.overlapping=False - self.overlapping_indices=0 - self.total_boxes=1 - with open('tests/inference_example.json') as f: - self.inference_exemple = json.load(f) - self.filename=self.inference_exemple["filename"] - self.labelOccurrence=self.inference_exemple["labelOccurrence"] - self.total_boxes=self.inference_exemple["totalBoxes"] - self.boxes=self.inference_exemple["boxes"] - - def test_build_inference_import(self): - """ - This test checks if the build_inference_import function returns a valid JSON object - """ - mock_inference = { - "filename": self.filename, - "labelOccurrence": self.labelOccurrence, - "totalBoxes": self.total_boxes, - } - inference_tested = inference.build_inference_import(self.inference_exemple) - inference_tested = json.loads(inference_tested) - self.assertGreaterEqual(len(mock_inference), len(inference_tested), "The inference returned has too many keys") - for key, value in mock_inference.items(): - self.assertTrue( key in inference_tested, f"{key} should be in the inference object") - self.assertEqual( - inference_tested[key], - value, - f"{key} should be {value}", - ) - - def test_build_object_import(self): - """ - This test checks if the build_object_import function returns a valid JSON object - """ - object =self.boxes[0] - object_tested = inference.build_object_import(object) - data = json.loads(object_tested) - mock_object = { - "box": { - "topX": 0.0, - "topY": 0.0, - "bottomX": 0.0, - "bottomY": 0.0 - }, - "color": "#ff0", - } - for key, value in mock_object.items(): - self.assertEqual( - data[key], - value, - f"{key} should be {value}", - ) - -class test_machine_learning_functions(unittest.TestCase): - def setUp(self): - with open("tests/ml_structure_exemple.json", "r") as file: - self.ml_structure = json.load(file) - self.model_name="that_model_name" - self.model_id="48efe646-5210-4761-908e-a06f95f0c344" - self.model_endpoint="https://that-model.inference.ml.com/score" - self.task="classification" - self.version=5 - self.data={ - "creation_date":"2024-01-01", - "created_by":"Avery GoodDataScientist", - "Accuracy": 0.6908 - } - self.mock_model={ - "task": "classification", - "endpoint": "https://that-model.inference.ml.com/score", - "api_key": "SeCRetKeys", - "content_type": "application/json", - "deployment_platform": "azure", - "endpoint_name": "that-model-endpoint", - "model_name": "that_model_name", - "model_id":"48efe646-5210-4761-908e-a06f95f0c344", - "created_by": "Avery GoodDataScientist", - "creation_date": "2024-01-01", - "version": 5, - "description": "Model Description", - "job_name": "Job Name", - "dataset": "Dataset Description", - "Accuracy": 0.6908 - } - self.pipeline_name="First Pipeline" - self.pipeline_id="48efe646-5210-4761-908e-a06f95f0c344" - self.pipeline_default=True - self.model_ids=["that_model_name", "other_model_name"] - self.mock_pipeline={ - "models": ["that_model_name", "other_model_name"], - "pipeline_id": "48efe646-5210-4761-908e-a06f95f0c344", - "pipeline_name": "First Pipeline", - "model_name": "First Pipeline", - "created_by": "Avery GoodDataScientist", - "creation_date": "2024-01-01", - "version": 1, - "description": "Pipeline Description", - "job_name": "Job Name", - "dataset": "Dataset Description", - "Accuracy": 0.6908, - "default": True - } - - def test_build_model_import(self): - """ - This test checks if the build_model_import function returns a valid JSON object - """ - model = self.ml_structure["models"][0] - model_import = ml_data.build_model_import(model) - model_data = json.loads(model_import) - self.assertLessEqual(len(model_data),len(self.mock_model), "The returned model data should have more key than expected") - for key in model_data: - self.assertEqual(self.mock_model[key], model_data[key], f"{key} should be {self.mock_model[key]}") - - def test_missing_key_build_model_import(self): - """ - This test checks if the build_model_import function raises an error when a key is missing - """ - model = self.ml_structure["models"][0] - model.pop("api_key") - self.assertRaises(ml_data.MissingKeyError, ml_data.build_model_import, model) - - def test_build_model_export(self): - """ - This test checks if the build_model_export function returns a valid JSON object - """ - model_export = ml_data.build_model_export(self.data,self.model_id,self.model_name,self.model_endpoint,self.task,self.version) - self.assertLessEqual(len(model_export),len(self.mock_model), "The returned model data should have more key than expected") - for key in model_export: - self.assertTrue(key in self.mock_model, f"The key:{key} was not expected") - self.assertEqual(self.mock_model[key], model_export[key], f"{key} should be {self.mock_model[key]}") - - def test_build_pipeline_import(self): - """ - This test checks if the build_pipeline_import function returns a valid JSON object - """ - pipeline = self.ml_structure["pipelines"][0] - pipeline_import = ml_data.build_pipeline_import(pipeline) - pipeline_data = json.loads(pipeline_import) - self.assertLessEqual(len(pipeline_data),len(self.mock_pipeline), "The returned pipeline data should have more key than expected") - for key in pipeline_data: - self.assertEqual(pipeline_data[key], self.mock_pipeline[key], f"The pipeline data for the key:{key} should be {self.mock_pipeline[key]}") - - def test_missing_key_build_pipeline_import(self): - """ - This test checks if the build_pipeline_import function raises an error when a key is missing - """ - pipeline = self.ml_structure["pipelines"][0] - pipeline.pop("models") - self.assertRaises(ml_data.MissingKeyError, ml_data.build_pipeline_import, pipeline) - - def test_build_pipeline_export(self): - """ - This test checks if the build_pipeline_export function returns a valid JSON object - """ - pipeline_export = ml_data.build_pipeline_export(self.data,self.pipeline_name,self.pipeline_id,self.pipeline_default,self.model_ids) - self.assertLessEqual(len(pipeline_export),len(self.mock_pipeline), "The returned pipeline data should have more key than expected") - for key in pipeline_export: - self.assertTrue(key in self.mock_pipeline, f"The key:{key} was not expected") - self.assertEqual(self.mock_pipeline[key], pipeline_export[key], f"{key} should be {self.mock_pipeline[key]}") if __name__ == "__main__": unittest.main()