diff --git a/app.py b/app.py index 8fa934d..da5192b 100644 --- a/app.py +++ b/app.py @@ -145,6 +145,9 @@ async def before_serving(): if CONNECTION_STRING is None: raise ServerError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") + if NACHET_DATA is None: + raise ServerError("Missing environment variable: NACHET_DATA") + if FERNET_KEY is None: raise ServerError("Missing environment variable: FERNET_KEY") @@ -176,11 +179,10 @@ async def before_serving(): """ ) #TODO Transform into logging - except (ServerError, inference.ModelAPIErrors) as e: + except (Exception, ServerError, inference.ModelAPIErrors) as e: print(e) raise - @app.post("/get-user-id") async def get_user_id() : """ @@ -321,7 +323,7 @@ async def delete_with_archive(): print(error) return jsonify([f"DeleteDirectoryRequestError: {str(error)}"]), 400 - +# Deprecated @app.post("/dir") async def list_directories(): """ @@ -346,6 +348,73 @@ async def list_directories(): print(error) return jsonify([f"ListDirectoriesRequestError: {str(error)}"]), 400 +@app.post("/get-directories") +async def get_directories(): + """ + get all directories in the user's container with pictures names and number of pictures + """ + try: + data = await request.get_json() + user_id = data["container_name"] + if user_id: + # Open db connection + connection = datastore.get_connection() + cursor = datastore.get_cursor(connection) + + directories_list = await datastore.get_directories(cursor, str(user_id)) + + # Close connection + datastore.end_query(connection, cursor) + + result = {"folders" : directories_list} + return jsonify(result) + else: + raise ListDirectoriesRequestError("Missing container name") + + except (KeyError, TypeError, ListDirectoriesRequestError, azure_storage.MountContainerError, datastore.DatastoreError) as error: + print(error) + return jsonify([f"ListDirectoriesRequestError: {str(error)}"]), 400 + +@app.post("/get-picture") +async def get_picture(): + """ + get all directories in the user's container with pictures names and number of pictures + """ + try: + data = await request.get_json() + container_name = data["container_name"] + user_id = container_name + picture_id = data["picture_id"] + + if user_id: + + container_client = await azure_storage.mount_container( + CONNECTION_STRING, container_name, create_container=True + ) + # Open db connection + connection = datastore.get_connection() + cursor = datastore.get_cursor(connection) + + picture = {} + picture["picture_id"] = picture_id + + inference = await datastore.get_inference(cursor, str(user_id), str(picture_id)) + picture["inference"] = inference + + blob = await datastore.get_picture_blob(cursor, str(user_id), container_client, str(picture_id)) + image_base64 = base64.b64encode(blob) + picture["image"] = "data:image/tiff;base64," + image_base64.decode("utf-8") + + # Close connection + datastore.end_query(connection, cursor) + return jsonify(picture) + else: + raise ListDirectoriesRequestError("Missing container name") + + except (KeyError, TypeError, ListDirectoriesRequestError, azure_storage.MountContainerError, datastore.DatastoreError) as error: + print(error) + return jsonify([f"ListDirectoriesRequestError: {str(error)}"]), 400 + @app.post("/create-dir") async def create_directory(): @@ -371,7 +440,7 @@ async def create_directory(): if response: return jsonify([response]), 200 else: - raise CreateDirectoryRequestError("directory already exists") + raise CreateDirectoryRequestError("Error while creating directory") else: raise CreateDirectoryRequestError("missing container or directory name") @@ -487,9 +556,8 @@ async def inference_request(): connection = datastore.get_connection() cursor = datastore.get_cursor(connection) - image_hash_value = await azure_storage.generate_hash(image_bytes) picture_id = await datastore.get_picture_id( - cursor, user_id, image_hash_value, container_client + cursor, user_id, image_bytes, container_client ) # Close connection datastore.end_query(connection, cursor) @@ -508,17 +576,6 @@ async def inference_request(): cache_json_result[-1], imageDims, area_ratio, color_format ) - result_json_string = await record_model(pipeline, processed_result_json) - - # upload the inference results to the user's container as async task - app.add_background_task( - azure_storage.upload_inference_result, - container_client, - folder_name, - result_json_string, - image_hash_value, - ) - # Open db connection connection = datastore.get_connection() cursor = datastore.get_cursor(connection) diff --git a/docs/nachet-manage-folders.md b/docs/nachet-manage-folders.md index 8bfab1b..1cc7bbe 100644 --- a/docs/nachet-manage-folders.md +++ b/docs/nachet-manage-folders.md @@ -145,10 +145,49 @@ note left of FE : "Are you sure ? Everything in this folder will be deleted and The `create-dir` route need a folder_name and create the folder in database and in Azure Blob storage. -### /dir +### /get-directories + +The `get-directories` route retreives all user directories from the database +with their pictures as a json. There is 4 different cases for the pictures : + +| **is_verified \\ inference_exist** | **false** | **true** | +|------------------------------------|----------------------|-----------------------| +| **false** | *should not happend* | inference not verified | +| **true** | batch import | inference verified | + +```json +{ +"folders" : [ + { + "picture_set_id" : "xxxx-xxxx-xxxx-xxxx", + "folder_name" : "folder name", + "nb_pictures": 4, + "pictures" : [ + { + "picture_id" : "xxxx-xxxx-xxxx-xxxx", + "inference_exist": false, + "is_validated": true + }, + ... + ] + }, + ... + ] +} +``` + +### /get-picture -The `dir` route retreives all user directories from the database (id, name and -nb_pictures). +The `get-picture` route retreives selected picture as a json : + +```json +{ + "picture_id" : "xxxx-xxxx-xxxx-xxxx", + "inference": { + } + "image": "data:image/...;base64,xxxxxxxx" +} +``` ### /delete-request diff --git a/model/test.py b/model/test.py index 2221e56..4447637 100644 --- a/model/test.py +++ b/model/test.py @@ -50,7 +50,14 @@ async def request_inference_from_test(model: namedtuple, previous_result: str): }, ], } - ] + ], + "models" : + [ + { + "name" : model.name, + "version" : 1 + } + ] } ] diff --git a/storage/datastore_storage_api.py b/storage/datastore_storage_api.py index 8ea2b41..cb7dcc3 100644 --- a/storage/datastore_storage_api.py +++ b/storage/datastore_storage_api.py @@ -26,6 +26,12 @@ class UserNotFoundError(DatastoreError): NACHET_DB_URL = os.getenv("NACHET_DB_URL") NACHET_SCHEMA = os.getenv("NACHET_SCHEMA") +if NACHET_DB_URL is None: + raise DatastoreError("Missing environment variable: NACHET_DB_URL") + +if NACHET_SCHEMA is None: + raise DatastoreError("Missing environment variable: NACHET_SCHEMA") + def get_connection() : return db.connect_db(NACHET_DB_URL, NACHET_SCHEMA) @@ -90,11 +96,11 @@ async def create_user(email: str, connection_string) -> datastore.User: return user -async def get_picture_id(cursor, user_id, image_hash_value, container_client) : +async def get_picture_id(cursor, user_id, image, container_client) : """ Return the picture_id of the image """ - picture_id = await nachet_datastore.upload_picture_unknown(cursor, str(user_id), image_hash_value, container_client) + picture_id = await nachet_datastore.upload_picture_unknown(cursor, str(user_id), image, container_client) return picture_id def upload_pictures(cursor, user_id, picture_set_id, container_client, pictures, seed_name: str, zoom_level: float = None, nb_seeds: int = None) : @@ -151,6 +157,18 @@ async def delete_directory_with_archive(cursor, user_id, picture_set_id, contain async def get_directories(cursor, user_id): try : - return await datastore.get_picture_sets_info(cursor, user_id) + return await nachet_datastore.get_picture_sets_info(cursor, user_id) + except Exception as error: + raise DatastoreError(error) + +async def get_inference(cursor, user_id, picture_id): + try : + return await nachet_datastore.get_picture_inference(cursor, user_id, picture_id) + except Exception as error: + raise DatastoreError(error) + +async def get_picture_blob(cursor, user_id, container_client, picture_id): + try : + return await nachet_datastore.get_picture_blob(cursor, user_id, container_client, picture_id) except Exception as error: raise DatastoreError(error) diff --git a/tests/test_manage_folders.py b/tests/test_manage_folders.py new file mode 100644 index 0000000..f5d001e --- /dev/null +++ b/tests/test_manage_folders.py @@ -0,0 +1,358 @@ +import os +import unittest +import asyncio +import json +import base64 +from unittest.mock import patch, MagicMock +from app import app +from storage.datastore_storage_api import DatastoreError + +class TestMissingEnvError(Exception): + pass + +CONNECTION_STRING = os.getenv("NACHET_AZURE_STORAGE_CONNECTION_STRING") +NACHET_DB_URL = os.getenv("NACHET_DB_URL") +NACHET_SCHEMA = os.getenv("NACHET_SCHEMA") + +if CONNECTION_STRING is None: + raise TestMissingEnvError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") +if NACHET_DB_URL is None: + raise TestMissingEnvError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") +if NACHET_SCHEMA is None: + raise TestMissingEnvError("Missing environment variable: NACHET_AZURE_STORAGE_CONNECTION_STRING") + +class TestCreateFolder(unittest.TestCase): + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + self.container_name = "test_container_name" + self.folder_name = "test_folder_name" + self.picture_set_id = "picture_set_id" + + # Mock the azure_storage and database variables + self.mock_cur = MagicMock() + self.mock_connection = MagicMock() + self.mock_container_client = MagicMock() + + # Patch the azure_storage and datastore functions + self.patch_connect_db = patch('app.datastore.db.connect_db', return_value=self.mock_connection) + self.patch_cursor = patch('app.datastore.db.cursor', return_value=self.mock_cur) + self.patch_mount_container = patch('app.azure_storage.mount_container', return_value=self.mock_container_client) + self.patch_create_picture_set = patch('app.datastore.create_picture_set', return_value = self.picture_set_id) + self.patch_end_query = patch('app.datastore.end_query') + + self.mock_connect_db = self.patch_connect_db.start() + self.mock_cursor = self.patch_cursor.start() + self.mock_mount_container = self.patch_mount_container.start() + self.mock_create_picture_set = self.patch_create_picture_set.start() + self.mock_end_query = self.patch_end_query.start() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + self.patch_connect_db.stop() + self.patch_cursor.stop() + self.patch_mount_container.stop() + self.patch_create_picture_set.stop() + self.patch_end_query.stop() + + + def test_create_directory_successfull(self): + """ + Test the directory creation route with successful conditions. + """ + response = asyncio.run( + self.test_client.post( + '/create-dir', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "folder_name": self.folder_name + }) + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(json.loads(asyncio.run(response.get_data())), [self.picture_set_id]) + self.mock_mount_container.assert_called_once_with(CONNECTION_STRING, self.container_name, create_container=True) + self.mock_connect_db.assert_called_once_with(NACHET_DB_URL, NACHET_SCHEMA) + self.mock_cursor.assert_called_once_with(self.mock_connection) + self.mock_create_picture_set.assert_called_once_with(self.mock_cur, self.mock_container_client, self.container_name, 0, self.folder_name) + self.mock_end_query.assert_called_once_with(self.mock_connection, self.mock_cur) + + def test_create_directory_missing_argument_error(self): + """ + Test the directory creation route with unsuccessful conditions : missing argument. + """ + expected = ("CreateDirectoryRequestError: missing container or directory name") + + response = asyncio.run( + self.test_client.post( + '/create-dir', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "folder_name": "" + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + + def test_create_directory_datastore_error(self): + """ + Test the directory creation route with unsuccessful conditions : an error from datastore is raised. + """ + expected = ("CreateDirectoryRequestError: An error occured during the upload of the picture set") + self.mock_create_picture_set.side_effect = DatastoreError("An error occured during the upload of the picture set") + + response = asyncio.run( + self.test_client.post( + '/create-dir', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "folder_name": self.folder_name + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + +class TestGetFolders(unittest.TestCase): + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + self.container_name = "test_container_name" + self.folders_data = [ + { + "folder_name": "General", + "nb_pictures": 1, + "picture_set_id": "picture_set_id", + "pictures": [{ + "inference_exist": True, + "is_validated": False, + "picture_id": "picture_id" + }] + } + ] + + # Mock the azure_storage and database variables + self.mock_cur = MagicMock() + self.mock_connection = MagicMock() + + # Patch the azure_storage and datastore functions + self.patch_connect_db = patch('app.datastore.db.connect_db', return_value=self.mock_connection) + self.patch_cursor = patch('app.datastore.db.cursor', return_value=self.mock_cur) + self.patch_get_directories = patch('app.datastore.get_directories', return_value = self.folders_data) + self.patch_end_query = patch('app.datastore.end_query') + + self.mock_connect_db = self.patch_connect_db.start() + self.mock_cursor = self.patch_cursor.start() + self.mock_get_directories = self.patch_get_directories.start() + self.mock_end_query = self.patch_end_query.start() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + self.patch_connect_db.stop() + self.patch_cursor.stop() + self.patch_get_directories.stop() + self.patch_end_query.stop() + + def test_get_directories_successfull(self): + """ + Test the get directories route with successful conditions. + """ + response = asyncio.run( + self.test_client.post( + '/get-directories', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name + }) + ) + + self.assertEqual(response.status_code, 200) + self.assertDictEqual(json.loads(asyncio.run(response.get_data())), {"folders" : self.folders_data}) + self.mock_connect_db.assert_called_once_with(NACHET_DB_URL, NACHET_SCHEMA) + self.mock_cursor.assert_called_once_with(self.mock_connection) + self.mock_get_directories.assert_called_once_with(self.mock_cur, self.container_name) + self.mock_end_query.assert_called_once_with(self.mock_connection, self.mock_cur) + + def test_get_directories_missing_argument_error(self): + """ + Test the get directories route with unsuccessful conditions : missing argument. + """ + expected = ("ListDirectoriesRequestError: Missing container name") + + response = asyncio.run( + self.test_client.post( + '/get-directories', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": "" + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + + def test_get_directories_datastore_error(self): + """ + Test the get directories route with unsuccessful conditions : an error from datastore is raised. + """ + expected = ("ListDirectoriesRequestError: An error occured while retrieving the picture sets") + self.mock_get_directories.side_effect = DatastoreError("An error occured while retrieving the picture sets") + + response = asyncio.run( + self.test_client.post( + '/get-directories', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name + }) + ) + + self.assertEqual(response.status_code, 400) + result_json = json.loads(asyncio.run(response.get_data())) + self.assertEqual(result_json[0], expected) + + +class TestGetPicture(unittest.TestCase): + + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + self.container_name = "test_container_name" + self.picture_id = "test_picture_id" + self.inference = { + "boxes": [ + { + "box_id": "test_box_id", + "label": "test_label", + "score": 1, + "top_id": "test_top_id" + } + ], + "inference_id": "test_inference_id", + "models": [ + { + "name": "test_model_name", + "version": "1" + }, + ], + "pipeline_id": "test_pipeline_id", + } + self.picture_blob = b"blob" + image_base64 = base64.b64encode(self.picture_blob) + self.image = "data:image/tiff;base64," + image_base64.decode("utf-8") + + # Mock the azure_storage and database variables + self.mock_cur = MagicMock() + self.mock_connection = MagicMock() + self.mock_container_client = MagicMock() + + # Patch the azure_storage and datastore functions + self.patch_connect_db = patch('app.datastore.db.connect_db', return_value=self.mock_connection) + self.patch_cursor = patch('app.datastore.db.cursor', return_value=self.mock_cur) + self.patch_mount_container = patch('app.azure_storage.mount_container', return_value=self.mock_container_client) + self.patch_get_inference = patch('app.datastore.get_inference', return_value = self.inference) + self.patch_get_picture_blob = patch('app.datastore.get_picture_blob', return_value = self.picture_blob) + self.patch_end_query = patch('app.datastore.end_query') + + self.mock_connect_db = self.patch_connect_db.start() + self.mock_cursor = self.patch_cursor.start() + self.mock_mount_container = self.patch_mount_container.start() + self.mock_get_inference = self.patch_get_inference.start() + self.mock_get_picture_blob = self.patch_get_picture_blob.start() + self.mock_end_query = self.patch_end_query.start() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + self.patch_connect_db.stop() + self.patch_cursor.stop() + self.patch_mount_container.stop() + self.patch_get_inference.stop() + self.patch_get_picture_blob.stop() + self.patch_end_query.stop() + + def test_get_picture_successfull(self): + """ + Test the get picture route with successful conditions. + """ + response = asyncio.run( + self.test_client.post( + '/get-picture', + headers={ + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + json={ + "container_name": self.container_name, + "picture_id": self.picture_id + }) + ) + + self.assertEqual(response.status_code, 200) + self.maxDiff = None + self.assertDictEqual(json.loads(asyncio.run(response.get_data())), {"inference": self.inference, "image": self.image, "picture_id": self.picture_id}) + self.mock_mount_container.assert_called_once_with(CONNECTION_STRING, self.container_name, create_container=True) + self.mock_connect_db.assert_called_once_with(NACHET_DB_URL, NACHET_SCHEMA) + self.mock_cursor.assert_called_once_with(self.mock_connection) + self.mock_get_inference.assert_called_once_with(self.mock_cur, self.container_name, self.picture_id) + self.mock_get_picture_blob.assert_called_once_with(self.mock_cur, self.container_name, self.mock_container_client, self.picture_id) + self.mock_end_query.assert_called_once_with(self.mock_connection, self.mock_cur) + + +class TestDeleteFolder(unittest.TestCase): + + #TODO: implement the tests for the delete folder route + + def setUp(self) -> None: + """ + Set up the test environment before running each test case. + """ + self.test_client = app.test_client() + + def tearDown(self) -> None: + """ + Tear down the test environment at the end of each test case. + """ + self.test_client = None + +if __name__ == '__main__': + unittest.main()