From 8a8fce7e88994eac05baf6e230ad13ea301a05a0 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 17 Jul 2024 20:03:47 +0000 Subject: [PATCH 1/8] update manage folder doc #105 --- docs/nachet-manage-folders.md | 50 ++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/nachet-manage-folders.md b/docs/nachet-manage-folders.md index 8bfab1b..c02c542 100644 --- a/docs/nachet-manage-folders.md +++ b/docs/nachet-manage-folders.md @@ -145,10 +145,54 @@ note left of FE : "Are you sure ? Everything in this folder will be deleted and The `create-dir` route need a folder_name and create the folder in database and in Azure Blob storage. -### /dir +### /directories + +The `directories` route retreives all user directories from the database with their pictures as a json. There is 4 different cases for the pictures : + +| **is_verified \\ inference_exist** | **false** | **true** | +|------------------------------------|----------------------|-----------------------| +| **false** | ? | inference not verified | +| **true** | batch import | inference verified | + +```json +{ +"folders" : [ + { + "picture_set_id" : "xxxx-xxxx-xxxx-xxxx", + "folder_name" : "folder name", + "nb_pictures": 4, + "pictures" : [ + { + "picture_id" : "xxxx-xxxx-xxxx-xxxx", + "inference_exist": false, + "is_validated": true + }, + ... + ] + }, + ... + ] +} +``` + +### /get-folder-content + +The `get-folder-content` route saves inferences and picture blobs in the cache and return nothing. -The `dir` route retreives all user directories from the database (id, name and -nb_pictures). +### /get-picture + +The `get-picture` route retreives selected picture as a json : + +```json +{ + "picture_id" : "xxxx-xxxx-xxxx-xxxx", + "inference_exist": true, + "is_validated": false, + "inference": { + } + "image": "xxxxxxxxxxx" +} +``` ### /delete-request From e2e7d44cebd255fe1ad2c18593ac54bdb16dad64 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Fri, 26 Jul 2024 15:01:08 +0000 Subject: [PATCH 2/8] get picture route #105 --- app.py | 71 +++++++++++++++++++++++++++++++- requirements.txt | 2 +- storage/datastore_storage_api.py | 12 ++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 8fa934d..b7427ed 100644 --- a/app.py +++ b/app.py @@ -1,3 +1,4 @@ +from tkinter import image_names import urllib.request import json import os @@ -176,10 +177,14 @@ async def before_serving(): """ ) #TODO Transform into logging - except (ServerError, inference.ModelAPIErrors) as e: + except (Exception, ServerError, inference.ModelAPIErrors) as e: print(e) raise +@app.before_request +def log_request_info(): + print(f"Call:{request.path}") + @app.post("/get-user-id") async def get_user_id() : @@ -321,7 +326,7 @@ async def delete_with_archive(): print(error) return jsonify([f"DeleteDirectoryRequestError: {str(error)}"]), 400 - +# Deprecated @app.post("/dir") async def list_directories(): """ @@ -346,6 +351,68 @@ async def list_directories(): print(error) return jsonify([f"ListDirectoriesRequestError: {str(error)}"]), 400 +@app.post("/get-directories") +async def get_directories(): + """ + get all directories in the user's container with pictures names and number of pictures + """ + try: + data = await request.get_json() + user_id = data["container_name"] + if user_id: + # Open db connection + connection = datastore.get_connection() + cursor = datastore.get_cursor(connection) + + directories_list = await datastore.get_directories(cursor, str(user_id)) + + # Close connection + datastore.end_query(connection, cursor) + + result = {"folders" : directories_list} + return jsonify(result) + else: + raise ListDirectoriesRequestError("Missing container name") + + except (KeyError, TypeError, ListDirectoriesRequestError, azure_storage.MountContainerError, datastore.DatastoreError) as error: + print(error) + return jsonify([f"ListDirectoriesRequestError: {str(error)}"]), 400 + +@app.post("/get-picture") +async def get_picture(): + """ + get all directories in the user's container with pictures names and number of pictures + """ + try: + data = await request.get_json() + user_id = data["container_name"] + picture_id = data["picture_id"] + + if user_id: + # Open db connection + connection = datastore.get_connection() + cursor = datastore.get_cursor(connection) + + picture = {} + picture["picture_id"] = picture_id + + inference = await datastore.get_inference(cursor, str(user_id), str(picture_id)) + picture["inference"] = inference + + image = await datastore.get_image_hash_value(cursor, str(user_id), str(picture_id)) + print(image) + picture["image"] = image + + # Close connection + datastore.end_query(connection, cursor) + return jsonify(picture) + else: + raise ListDirectoriesRequestError("Missing container name") + + except (KeyError, TypeError, ListDirectoriesRequestError, azure_storage.MountContainerError, datastore.DatastoreError) as error: + print(error) + return jsonify([f"ListDirectoriesRequestError: {str(error)}"]), 400 + @app.post("/create-dir") async def create_directory(): diff --git a/requirements.txt b/requirements.txt index 0805835..9429ac1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ cryptography pyyaml pydantic python-magic -nachet-datastore @git+https://github.com/ai-cfia/nachet-datastore.git@b0d85ce68ed08928b7240c35d9d462cda48ba294 +nachet-datastore @git+https://github.com/ai-cfia/nachet-datastore.git@a06703a6d32da71c851469ada1d5a4e5cfb3eda9 diff --git a/storage/datastore_storage_api.py b/storage/datastore_storage_api.py index 0c4f944..ccbbdf4 100644 --- a/storage/datastore_storage_api.py +++ b/storage/datastore_storage_api.py @@ -154,3 +154,15 @@ async def get_directories(cursor, user_id): return await datastore.get_picture_sets_info(cursor, user_id) except Exception as error: raise DatastoreError(error) + +async def get_inference(cursor, user_id, picture_id): + try : + return await nachet_datastore.get_picture_inference(cursor, user_id, picture_id) + except Exception as error: + raise DatastoreError(error) + +async def get_image_hash_value(cursor, user_id, picture_id): + try : + return await nachet_datastore.get_picture_blob(cursor, user_id, picture_id) + except Exception as error: + raise DatastoreError(error) \ No newline at end of file From 0786583f6323f304705fdcf72c740b5ea494e00b Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Mon, 29 Jul 2024 17:55:10 +0000 Subject: [PATCH 3/8] get-picture route fixes --- app.py | 16 ++++++++-------- requirements.txt | 2 +- storage/datastore_storage_api.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index b7427ed..b467a02 100644 --- a/app.py +++ b/app.py @@ -181,11 +181,6 @@ async def before_serving(): print(e) raise -@app.before_request -def log_request_info(): - print(f"Call:{request.path}") - - @app.post("/get-user-id") async def get_user_id() : """ @@ -384,11 +379,16 @@ async def get_picture(): get all directories in the user's container with pictures names and number of pictures """ try: - data = await request.get_json() - user_id = data["container_name"] + data = await request.get_json() + container_name = data["container_name"] + user_id = container_name picture_id = data["picture_id"] if user_id: + + container_client = await azure_storage.mount_container( + CONNECTION_STRING, container_name, create_container=True + ) # Open db connection connection = datastore.get_connection() cursor = datastore.get_cursor(connection) @@ -399,7 +399,7 @@ async def get_picture(): inference = await datastore.get_inference(cursor, str(user_id), str(picture_id)) picture["inference"] = inference - image = await datastore.get_image_hash_value(cursor, str(user_id), str(picture_id)) + image = await datastore.get_image_hash_value(cursor, str(user_id), container_client, str(picture_id)) print(image) picture["image"] = image diff --git a/requirements.txt b/requirements.txt index 949246c..3f1a4d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ cryptography pyyaml pydantic python-magic -nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@a06703a6d32da71c851469ada1d5a4e5cfb3eda9 +nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@0b76149def3a83220703689d3a10439809773399 diff --git a/storage/datastore_storage_api.py b/storage/datastore_storage_api.py index ccbbdf4..6c64b4d 100644 --- a/storage/datastore_storage_api.py +++ b/storage/datastore_storage_api.py @@ -151,7 +151,7 @@ async def delete_directory_with_archive(cursor, user_id, picture_set_id, contain async def get_directories(cursor, user_id): try : - return await datastore.get_picture_sets_info(cursor, user_id) + return await nachet_datastore.get_picture_sets_info(cursor, user_id) except Exception as error: raise DatastoreError(error) @@ -161,8 +161,8 @@ async def get_inference(cursor, user_id, picture_id): except Exception as error: raise DatastoreError(error) -async def get_image_hash_value(cursor, user_id, picture_id): +async def get_image_hash_value(cursor, user_id, container_client, picture_id): try : - return await nachet_datastore.get_picture_blob(cursor, user_id, picture_id) + return await nachet_datastore.get_picture_blob(cursor, user_id, container_client, picture_id) except Exception as error: raise DatastoreError(error) \ No newline at end of file From edf265f942907ddb67ddba06b12ec92d9036f313 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Tue, 30 Jul 2024 14:45:53 +0000 Subject: [PATCH 4/8] remove picture hash --- app.py | 14 +++++++------- requirements.txt | 2 +- storage/datastore_storage_api.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index b467a02..62f4320 100644 --- a/app.py +++ b/app.py @@ -399,9 +399,9 @@ async def get_picture(): inference = await datastore.get_inference(cursor, str(user_id), str(picture_id)) picture["inference"] = inference - image = await datastore.get_image_hash_value(cursor, str(user_id), container_client, str(picture_id)) - print(image) - picture["image"] = image + blob = await datastore.get_picture_blob(cursor, str(user_id), container_client, str(picture_id)) + image_base64 = base64.b64encode(blob) + picture["image"] = "data:image/tiff;base64," + image_base64.decode("utf-8") # Close connection datastore.end_query(connection, cursor) @@ -554,9 +554,8 @@ async def inference_request(): connection = datastore.get_connection() cursor = datastore.get_cursor(connection) - image_hash_value = await azure_storage.generate_hash(image_bytes) picture_id = await datastore.get_picture_id( - cursor, user_id, image_hash_value, container_client + cursor, user_id, image_bytes, container_client ) # Close connection datastore.end_query(connection, cursor) @@ -577,15 +576,16 @@ async def inference_request(): result_json_string = await record_model(pipeline, processed_result_json) + """ # upload the inference results to the user's container as async task app.add_background_task( azure_storage.upload_inference_result, container_client, folder_name, result_json_string, - image_hash_value, + image_bytes, ) - + """ # Open db connection connection = datastore.get_connection() cursor = datastore.get_cursor(connection) diff --git a/requirements.txt b/requirements.txt index 3f1a4d0..b40fa18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ cryptography pyyaml pydantic python-magic -nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@0b76149def3a83220703689d3a10439809773399 +nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@c1f2c2e87f3502b23ff616ac9c24c5345a412984 diff --git a/storage/datastore_storage_api.py b/storage/datastore_storage_api.py index 6c64b4d..31a3f37 100644 --- a/storage/datastore_storage_api.py +++ b/storage/datastore_storage_api.py @@ -90,11 +90,11 @@ async def create_user(email: str, connection_string) -> datastore.User: return user -async def get_picture_id(cursor, user_id, image_hash_value, container_client) : +async def get_picture_id(cursor, user_id, image, container_client) : """ Return the picture_id of the image """ - picture_id = await nachet_datastore.upload_picture_unknown(cursor, str(user_id), image_hash_value, container_client) + picture_id = await nachet_datastore.upload_picture_unknown(cursor, str(user_id), image, container_client) return picture_id def upload_pictures(cursor, user_id, picture_set_id, container_client, pictures, seed_name: str, zoom_level: float = None, nb_seeds: int = None) : @@ -161,7 +161,7 @@ async def get_inference(cursor, user_id, picture_id): except Exception as error: raise DatastoreError(error) -async def get_image_hash_value(cursor, user_id, container_client, picture_id): +async def get_picture_blob(cursor, user_id, container_client, picture_id): try : return await nachet_datastore.get_picture_blob(cursor, user_id, container_client, picture_id) except Exception as error: From 565c1e173346947501508001a230f35dd776b396 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Tue, 30 Jul 2024 20:42:44 +0000 Subject: [PATCH 5/8] test folders routes --- app.py | 17 +- storage/datastore_storage_api.py | 6 + tests/test_manage_folders.py | 358 +++++++++++++++++++++++++++++++ 3 files changed, 368 insertions(+), 13 deletions(-) create mode 100644 tests/test_manage_folders.py diff --git a/app.py b/app.py index 62f4320..e55fd4f 100644 --- a/app.py +++ b/app.py @@ -146,6 +146,9 @@ async def before_serving(): if CONNECTION_STRING is None: raise ServerError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") + if NACHET_DATA is None: + raise ServerError("Missing environment variable: NACHET_DATA") + if FERNET_KEY is None: raise ServerError("Missing environment variable: FERNET_KEY") @@ -438,7 +441,7 @@ async def create_directory(): if response: return jsonify([response]), 200 else: - raise CreateDirectoryRequestError("directory already exists") + raise CreateDirectoryRequestError("Error while creating directory") else: raise CreateDirectoryRequestError("missing container or directory name") @@ -574,18 +577,6 @@ async def inference_request(): cache_json_result[-1], imageDims, area_ratio, color_format ) - result_json_string = await record_model(pipeline, processed_result_json) - - """ - # upload the inference results to the user's container as async task - app.add_background_task( - azure_storage.upload_inference_result, - container_client, - folder_name, - result_json_string, - image_bytes, - ) - """ # Open db connection connection = datastore.get_connection() cursor = datastore.get_cursor(connection) diff --git a/storage/datastore_storage_api.py b/storage/datastore_storage_api.py index 31a3f37..ec0554f 100644 --- a/storage/datastore_storage_api.py +++ b/storage/datastore_storage_api.py @@ -26,6 +26,12 @@ class UserNotFoundError(DatastoreError): NACHET_DB_URL = os.getenv("NACHET_DB_URL") NACHET_SCHEMA = os.getenv("NACHET_SCHEMA") +if NACHET_DB_URL is None: + raise DatastoreError("Missing environment variable: NACHET_DB_URL") + +if NACHET_SCHEMA is None: + raise DatastoreError("Missing environment variable: NACHET_SCHEMA") + def get_connection() : return db.connect_db(NACHET_DB_URL, NACHET_SCHEMA) diff --git a/tests/test_manage_folders.py b/tests/test_manage_folders.py new file mode 100644 index 0000000..2f36788 --- /dev/null +++ b/tests/test_manage_folders.py @@ -0,0 +1,358 @@ +import os +import unittest +import asyncio +import json +import base64 +from unittest.mock import patch, MagicMock +from app import app +from storage.datastore_storage_api import DatastoreError + +class TestMissingEnvError(Exception): + pass + +CONNECTION_STRING = os.getenv("NACHET_AZURE_STORAGE_CONNECTION_STRING") +NACHET_DB_URL = os.getenv("NACHET_DB_URL") +NACHET_SCHEMA = os.getenv("NACHET_SCHEMA") + +if CONNECTION_STRING is None: + raise TestMissingEnvError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") +if NACHET_DB_URL is None: + raise TestMissingEnvError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") +if NACHET_SCHEMA is None: + raise TestMissingEnvError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") + +class TestCreateFolder(unittest.TestCase): + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + self.container_name = "test_container_name" + self.folder_name = "test_folder_name" + self.picture_set_id = "picture_set_id" + + # Mock the azure_storage and database variables + self.mock_cur = MagicMock() + self.mock_connection = MagicMock() + self.mock_container_client = MagicMock() + + # Patch the azure_storage and datastore functions + self.patch_connect_db = patch('app.datastore.db.connect_db', return_value=self.mock_connection) + self.patch_cursor = patch('app.datastore.db.cursor', return_value=self.mock_cur) + self.patch_mount_container = patch('app.azure_storage.mount_container', return_value=self.mock_container_client) + self.patch_create_picture_set = patch('app.datastore.create_picture_set', return_value = self.picture_set_id) + self.patch_end_query = patch('app.datastore.end_query') + + self.mock_connect_db = self.patch_connect_db.start() + self.mock_cursor = self.patch_cursor.start() + self.mock_mount_container = self.patch_mount_container.start() + self.mock_create_picture_set = self.patch_create_picture_set.start() + self.mock_end_query = self.patch_end_query.start() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + self.patch_connect_db.stop() + self.patch_cursor.stop() + self.patch_mount_container.stop() + self.patch_create_picture_set.stop() + self.patch_end_query.stop() + + + def test_create_directory_successfull(self): + """ + Test the directory creation route with successful conditions. + """ + response = asyncio.run( + self.test_client.post( + '/create-dir', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "folder_name": self.folder_name + }) + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(asyncio.run(response.get_data())), [self.picture_set_id]) + self.mock_mount_container.assert_called_once_with(CONNECTION_STRING, self.container_name, create_container=True) + self.mock_connect_db.assert_called_once_with(NACHET_DB_URL, NACHET_SCHEMA) + self.mock_cursor.assert_called_once_with(self.mock_connection) + self.mock_create_picture_set.assert_called_once_with(self.mock_cur, self.mock_container_client, self.container_name, 0, self.folder_name) + self.mock_end_query.assert_called_once_with(self.mock_connection, self.mock_cur) + + def test_create_directory_missing_argument_error(self): + """ + Test the directory creation route with unsuccessful conditions : missing argument. + """ + expected = ("CreateDirectoryRequestError: missing container or directory name") + + response = asyncio.run( + self.test_client.post( + '/create-dir', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "folder_name": "" + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + + def test_create_directory_datastore_error(self): + """ + Test the directory creation route with unsuccessful conditions : an error from datastore is raised. + """ + expected = ("CreateDirectoryRequestError: An error occured during the upload of the picture set") + self.mock_create_picture_set.side_effect = DatastoreError("An error occured during the upload of the picture set") + + response = asyncio.run( + self.test_client.post( + '/create-dir', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "folder_name": self.folder_name + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + +class TestGetFolders(unittest.TestCase): + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + self.container_name = "test_container_name" + self.folders_data = [ + { + "folder_name": "General", + "nb_pictures": 1, + "picture_set_id": "picture_set_id", + "pictures": [{ + "inference_exist": True, + "is_validated": False, + "picture_id": "picture_id" + }] + } + ] + + # Mock the azure_storage and database variables + self.mock_cur = MagicMock() + self.mock_connection = MagicMock() + + # Patch the azure_storage and datastore functions + self.patch_connect_db = patch('app.datastore.db.connect_db', return_value=self.mock_connection) + self.patch_cursor = patch('app.datastore.db.cursor', return_value=self.mock_cur) + self.patch_get_directories = patch('app.datastore.get_directories', return_value = self.folders_data) + self.patch_end_query = patch('app.datastore.end_query') + + self.mock_connect_db = self.patch_connect_db.start() + self.mock_cursor = self.patch_cursor.start() + self.mock_get_directories = self.patch_get_directories.start() + self.mock_end_query = self.patch_end_query.start() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + self.patch_connect_db.stop() + self.patch_cursor.stop() + self.patch_get_directories.stop() + self.patch_end_query.stop() + + def test_get_directories_successfull(self): + """ + Test the get directories route with successful conditions. + """ + response = asyncio.run( + self.test_client.post( + '/get-directories', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name + }) + ) + + self.assertEqual(response.status_code, 200) + self.assertDictEqual(json.loads(asyncio.run(response.get_data())), {"folders" : self.folders_data}) + self.mock_connect_db.assert_called_once_with(NACHET_DB_URL, NACHET_SCHEMA) + self.mock_cursor.assert_called_once_with(self.mock_connection) + self.mock_get_directories.assert_called_once_with(self.mock_cur, self.container_name) + self.mock_end_query.assert_called_once_with(self.mock_connection, self.mock_cur) + + def test_get_directories_missing_argument_error(self): + """ + Test the get directories route with unsuccessful conditions : missing argument. + """ + expected = ("ListDirectoriesRequestError: Missing container name") + + response = asyncio.run( + self.test_client.post( + '/get-directories', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": "" + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + + def test_get_directories_datastore_error(self): + """ + Test the get directories route with unsuccessful conditions : an error from datastore is raised. + """ + expected = ("ListDirectoriesRequestError: An error occured while retrieving the picture sets") + self.mock_get_directories.side_effect = DatastoreError("An error occured while retrieving the picture sets") + + response = asyncio.run( + self.test_client.post( + '/get-directories', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + + +class TestGetPicture(unittest.TestCase): + + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + self.container_name = "test_container_name" + self.picture_id = "test_picture_id" + self.inference = { + "boxes": [ + { + "box_id": "test_box_id", + "label": "test_label", + "score": 1, + "top_id": "test_top_id" + } + ], + "inference_id": "test_inference_id", + "models": [ + { + "name": "test_model_name", + "version": "1" + }, + ], + "pipeline_id": "test_pipeline_id", + } + self.picture_blob = b"blob" + image_base64 = base64.b64encode(self.picture_blob) + self.image = "data:image/tiff;base64," + image_base64.decode("utf-8") + + # Mock the azure_storage and database variables + self.mock_cur = MagicMock() + self.mock_connection = MagicMock() + self.mock_container_client = MagicMock() + + # Patch the azure_storage and datastore functions + self.patch_connect_db = patch('app.datastore.db.connect_db', return_value=self.mock_connection) + self.patch_cursor = patch('app.datastore.db.cursor', return_value=self.mock_cur) + self.patch_mount_container = patch('app.azure_storage.mount_container', return_value=self.mock_container_client) + self.patch_get_inference = patch('app.datastore.get_inference', return_value = self.inference) + self.patch_get_picture_blob = patch('app.datastore.get_picture_blob', return_value = self.picture_blob) + self.patch_end_query = patch('app.datastore.end_query') + + self.mock_connect_db = self.patch_connect_db.start() + self.mock_cursor = self.patch_cursor.start() + self.mock_mount_container = self.patch_mount_container.start() + self.mock_get_inference = self.patch_get_inference.start() + self.mock_get_picture_blob = self.patch_get_picture_blob.start() + self.mock_end_query = self.patch_end_query.start() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + self.patch_connect_db.stop() + self.patch_cursor.stop() + self.patch_mount_container.stop() + self.patch_get_inference.stop() + self.patch_get_picture_blob.stop() + self.patch_end_query.stop() + + def test_get_picture_successfull(self): + """ + Test the get picture route with successful conditions. + """ + response = asyncio.run( + self.test_client.post( + '/get-picture', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "picture_id": self.picture_id + }) + ) + + self.assertEqual(response.status_code, 200) + self.maxDiff = None + self.assertDictEqual(json.loads(asyncio.run(response.get_data())), {"inference": self.inference, "image": self.image, "picture_id": self.picture_id}) + self.mock_mount_container.assert_called_once_with(CONNECTION_STRING, self.container_name, create_container=True) + self.mock_connect_db.assert_called_once_with(NACHET_DB_URL, NACHET_SCHEMA) + self.mock_cursor.assert_called_once_with(self.mock_connection) + self.mock_get_inference.assert_called_once_with(self.mock_cur, self.container_name, self.picture_id) + self.mock_get_picture_blob.assert_called_once_with(self.mock_cur, self.container_name, self.mock_container_client, self.picture_id) + self.mock_end_query.assert_called_once_with(self.mock_connection, self.mock_cur) + + +class TestDeleteFolder(unittest.TestCase): + + #TODO: implement the tests for the delete folder route + + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 36dfeb5242d9341d9117ac509b463c4f9e3c5fb2 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 31 Jul 2024 15:10:18 +0000 Subject: [PATCH 6/8] fix tests --- model/test.py | 9 ++++++++- requirements.txt | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/model/test.py b/model/test.py index 2221e56..4447637 100644 --- a/model/test.py +++ b/model/test.py @@ -50,7 +50,14 @@ async def request_inference_from_test(model: namedtuple, previous_result: str): }, ], } - ] + ], + "models" : + [ + { + "name" : model.name, + "version" : 1 + } + ] } ] diff --git a/requirements.txt b/requirements.txt index b40fa18..dffc494 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ cryptography pyyaml pydantic python-magic -nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@c1f2c2e87f3502b23ff616ac9c24c5345a412984 +nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@f5595084c013e30c332b882c6135825386241147 \ No newline at end of file From 3ed39072a3e57093b489ea03d2fc72296255bf25 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Wed, 31 Jul 2024 15:28:11 +0000 Subject: [PATCH 7/8] fix lint errors --- app.py | 1 - docs/nachet-manage-folders.md | 15 +++++---------- requirements.txt | 2 +- storage/datastore_storage_api.py | 2 +- tests/test_manage_folders.py | 2 +- 5 files changed, 8 insertions(+), 14 deletions(-) diff --git a/app.py b/app.py index e55fd4f..da5192b 100644 --- a/app.py +++ b/app.py @@ -1,4 +1,3 @@ -from tkinter import image_names import urllib.request import json import os diff --git a/docs/nachet-manage-folders.md b/docs/nachet-manage-folders.md index c02c542..1cc7bbe 100644 --- a/docs/nachet-manage-folders.md +++ b/docs/nachet-manage-folders.md @@ -145,13 +145,14 @@ note left of FE : "Are you sure ? Everything in this folder will be deleted and The `create-dir` route need a folder_name and create the folder in database and in Azure Blob storage. -### /directories +### /get-directories -The `directories` route retreives all user directories from the database with their pictures as a json. There is 4 different cases for the pictures : +The `get-directories` route retreives all user directories from the database +with their pictures as a json. There is 4 different cases for the pictures : | **is_verified \\ inference_exist** | **false** | **true** | |------------------------------------|----------------------|-----------------------| -| **false** | ? | inference not verified | +| **false** | *should not happend* | inference not verified | | **true** | batch import | inference verified | ```json @@ -175,10 +176,6 @@ The `directories` route retreives all user directories from the database with th } ``` -### /get-folder-content - -The `get-folder-content` route saves inferences and picture blobs in the cache and return nothing. - ### /get-picture The `get-picture` route retreives selected picture as a json : @@ -186,11 +183,9 @@ The `get-picture` route retreives selected picture as a json : ```json { "picture_id" : "xxxx-xxxx-xxxx-xxxx", - "inference_exist": true, - "is_validated": false, "inference": { } - "image": "xxxxxxxxxxx" + "image": "data:image/...;base64,xxxxxxxx" } ``` diff --git a/requirements.txt b/requirements.txt index dffc494..1844a97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ cryptography pyyaml pydantic python-magic -nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@f5595084c013e30c332b882c6135825386241147 \ No newline at end of file +nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@f5595084c013e30c332b882c6135825386241147 diff --git a/storage/datastore_storage_api.py b/storage/datastore_storage_api.py index 66c181f..cb7dcc3 100644 --- a/storage/datastore_storage_api.py +++ b/storage/datastore_storage_api.py @@ -171,4 +171,4 @@ async def get_picture_blob(cursor, user_id, container_client, picture_id): try : return await nachet_datastore.get_picture_blob(cursor, user_id, container_client, picture_id) except Exception as error: - raise DatastoreError(error) \ No newline at end of file + raise DatastoreError(error) diff --git a/tests/test_manage_folders.py b/tests/test_manage_folders.py index 2f36788..f5d001e 100644 --- a/tests/test_manage_folders.py +++ b/tests/test_manage_folders.py @@ -355,4 +355,4 @@ def tearDown(self) -> None: self.test_client = None if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From 090675385cad49e2fab8de3c31e0a4c49b632247 Mon Sep 17 00:00:00 2001 From: sylvanie85 Date: Thu, 1 Aug 2024 17:06:41 +0000 Subject: [PATCH 8/8] datastore main branch --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1844a97..1291083 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ cryptography pyyaml pydantic python-magic -nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@f5595084c013e30c332b882c6135825386241147 +nachet-datastore @git+https://github.com/ai-cfia/ailab-datastore.git@main