diff --git a/app.py b/app.py index 46a9ca9..46800e0 100644 --- a/app.py +++ b/app.py @@ -169,7 +169,6 @@ async def before_serving(): # Store the seeds names and ml structure in CACHE CACHE["seeds"] = datastore.get_all_seeds_names() seeds = datastore.get_all_seeds() - print(jsonify(seeds)) CACHE["endpoints"] = await get_pipelines() print( @@ -398,6 +397,8 @@ async def inference_request(): picture_id = await datastore.get_picture_id( cursor, user_id, image_hash_value, container_client ) + # Close connection + datastore.end_query(connection, cursor) pipeline = pipelines_endpoints.get(pipeline_name) @@ -423,8 +424,14 @@ async def inference_request(): result_json_string, image_hash_value, ) + + # Open db connection + connection = datastore.get_connection() + cursor = datastore.get_cursor(connection) + saved_result_json = await datastore.save_inference_result(cursor, user_id, processed_result_json[0], picture_id, pipeline_name, 1) + # Close connection datastore.end_query(connection, cursor) # return the inference results to the client @@ -435,47 +442,6 @@ async def inference_request(): print(error) return jsonify(["InferenceRequestError: " + error.args[0]]), 400 -@app.get("/picture-form") -async def get_picture_form_info(): - """ - Retrieves the names of seeds from the database and returns them as a JSON - response. - - Returns: - A JSON response containing the names of seeds. - - Raises: - APIErrors: If there is an error while retrieving the seeds names from - the database. - """ - try: - seeds_names = datastore.get_all_seeds_names() - return jsonify(seeds_names), 200 - except datastore.DatastoreError as error: - return jsonify([error.args[0]]), 400 - -@app.put("/upload-pictures") -async def picture_batch_import(): - """ - This function handles the batch import of pictures. - - It performs the following steps: - 1. Uploads and chunks the file. - 2. Reconstructs the file and extracts data. - 3. Validates and uploads the data. - - Returns: - - If successful, returns a JSON response with the picture ID and a status code of 200. - - If an error occurs, returns a JSON response with the error message and a status code of 400. - """ - try: - temp_files = await upload_and_chunk_file(request) - email, picture_set, data = reconstruct_file_and_extract_data(temp_files) - picture_id = validate_and_upload_data(email, picture_set, data) - return jsonify([picture_id]), 200 - except APIErrors as error: - return jsonify([error.args[0]]), 400 - @app.get("/seed-data/") async def get_seed_data(seed_name): """ @@ -517,7 +483,6 @@ async def get_seeds(): Returns JSON containing the model seeds metadata """ seeds = await datastore.get_all_seeds() - print(jsonify(seeds)) if seeds : return jsonify(seeds), 200 else: @@ -559,69 +524,6 @@ async def record_model(pipeline: namedtuple, result: list): result[0]["models"] = new_entry return json.dumps(result, indent=4) -async def upload_and_chunk_file(request): - """ - Uploads a file and chunks it into smaller parts. - - Args: - request: The request object containing the file to be uploaded. - - Returns: - A list of file paths representing the chunks of the uploaded file. - """ - temp_dir = tempfile.TemporaryDirectory() - - upload_stream = await request.stream() - chunk_filename = os.path.join(temp_dir.name, f"chunk_{len(os.listdir(temp_dir.name))}") - with open(chunk_filename, "wb") as chunk_file: - async for chunk in upload_stream: - chunk_file.write(chunk) - - return [os.path.join(temp_dir.name, f) for f in os.listdir(temp_dir.name)] - -def reconstruct_file_and_extract_data(temp_files): - """ - Reconstructs a file from multiple chunks and extracts data from it. - - Args: - temp_files (list): A list of file paths to the temporary chunk files. - - Returns: - tuple: A tuple containing the extracted email, picture_set, and the original data. - """ - full_file = b'' - for chunk_filename in temp_files: - with open(chunk_filename, "rb") as chunk_file: - full_file += chunk_file.read() - data = json.loads(full_file) - email = data.get("email") - picture_set = data.get("picture_set") - return email, picture_set, data - -def validate_and_upload_data(email, picture_set, data): - """ - Validates the input parameters and uploads the picture set data to the database. - - Args: - email (str): The user's email address. - picture_set (list): The list of pictures in the picture set. - data (dict): Additional data for the picture set. - - Returns: - int: The ID of the uploaded picture set. - - Raises: - EmailNotSendError: If the user email is not provided. - EmptyPictureSetError: If no picture set is provided. - """ - if email is None: - raise EmailNotSendError("the user email is not provided") - if not picture_set: - raise EmptyPictureSetError("no picture set provided") - user_id = datastore.validate_user(email) - picture_id = datastore.upload_picture_set(user_id=user_id, **data) - return picture_id - async def fetch_json(repo_URL, key, file_path): """ Fetches JSON document from a GitHub repository. diff --git a/storage/datastore_storage_api.py b/storage/datastore_storage_api.py index 5c25e10..9cb5ebe 100644 --- a/storage/datastore_storage_api.py +++ b/storage/datastore_storage_api.py @@ -117,10 +117,6 @@ async def get_pipelines() -> list: raise GetPipelinesError(error.args[0]) async def save_inference_result(cursor, user_id:str, inference_dict, picture_id:str, pipeline_id:str, type:int): - nb_object = int(inference_dict["totalBoxes"]) - for box_index in range(nb_object): - print(inference_dict["boxes"][box_index]["label"]) - print(get_all_seeds()) return await datastore.register_inference_result(cursor, user_id, inference_dict, picture_id, pipeline_id, type) async def save_perfect_feedback(inference_id:str, user_id:str): diff --git a/tests/test_image_batch_import.py b/tests/test_image_batch_import.py deleted file mode 100644 index ae6f99b..0000000 --- a/tests/test_image_batch_import.py +++ /dev/null @@ -1,85 +0,0 @@ -import unittest -import asyncio - -from app import app, json -from unittest.mock import patch #Mock - - -class TestImageBatchImport(unittest.TestCase): - def setUp(self): - self.test_client = app.test_client() - - def tearDown(self): - pass - - @patch("datastore.db.queries.seed.get_all_seeds_names") - def test_picture_form_success(self, mock_seed_name): - - expected_result = ["seed_name"] - - mock_seed_name.return_value = ["seed_name"] - - response = asyncio.run(self.test_client.get("/picture-form")) - result = json.loads(asyncio.run(response.get_data())) - - self.assertEqual(response.status_code, 200) - self.assertEqual(result, expected_result) - - @patch("datastore.db.queries.seed.get_all_seeds_names") - def test_picture_form_failure(self, mock_seed_name): - - expected_result = "Error: seeds could not be retrieved" - - # TODO change with datastore.db.queries.seed.SeedNotFoundError() - mock_seed_name.side_effect = Exception("Error: seeds could not be retrieved") - - response = asyncio.run( - self.test_client.post( - "/picture-form", - headers={ - "Content-Type": "application/json", - "Access-Control-Allow-Origin": "*", - }, - json={"email": "test@inspection.gc.ca"} - )) - result = json.loads(asyncio.run(response.get_data())) - - self.assertEqual(response.status_code, 400) - self.assertEqual(result, expected_result) - - #@patch("datastore.bin.upload_picture_set.upload_picture_set") - def test_upload_picture_success(self): - - expected_result = 'a' * 300 * 1024* 1024 - - response = asyncio.run( - self.test_client.put( - "/upload-pictures", - headers={ - "Content-Type": "application/octet-stream", - "Access-Control-Allow-Origin": "*", - }, - data={"email": "test@email.com", - "nb_seeds": 6, - "seed_name": "seed_name", - "zoom_level": 1, - "picture_set": [expected_result] - } - )) - result = json.loads(asyncio.run(response.get_data())) - - self.assertEqual(response.status_code, 200) - self.assertEqual(result, expected_result) - - @patch("datastore.bin.upload_picture_set.upload_picture_set") - def test_upload_picture_failure(self, mock_upload_picture_set): - - expected_result = "An error occured during the upload of the picture set" - # TODO change with datastore.bin.upload_picture_set.UploadError() - mock_upload_picture_set.side_effect = Exception("An error occured during the upload of the picture set") - - response = asyncio.run(self.test_client.post("/upload-pictures")) - result = json.loads(asyncio.run(response.get_data())) - - self.assertEqual(response.status_code, 400) - self.assertEqual(result, expected_result) diff --git a/tests/test_inference_request.py b/tests/test_inference_request.py index a1c96b7..fcc7ef0 100644 --- a/tests/test_inference_request.py +++ b/tests/test_inference_request.py @@ -36,7 +36,7 @@ def tearDown(self) -> None: self.image_src = None self.test = None - @patch("azure_storage.azure_storage_api.mount_container") + @patch("storage.azure_storage_api.mount_container") def test_inference_request_successful(self, mock_container): # Mock azure client services mock_blob = Mock() @@ -78,6 +78,7 @@ def test_inference_request_successful(self, mock_container): "Access-Control-Allow-Origin": "*", }, json={ + "userId":"3e4d7d70-68d2-4302-a377-a869f1fd455e", "image": self.image_header + self.image_src, "imageDims": [720,540], "folder_name": self.folder_name, @@ -94,7 +95,7 @@ def test_inference_request_successful(self, mock_container): print(expected_keys == responses) self.assertEqual(responses, expected_keys) - @patch("azure_storage.azure_storage_api.mount_container") + @patch("storage.azure_storage_api.mount_container") def test_inference_request_unsuccessfull(self, mock_container): # Mock azure client services mock_blob = Mock() @@ -123,6 +124,7 @@ def test_inference_request_unsuccessfull(self, mock_container): "Access-Control-Allow-Origin": "*", }, json={ + "userId":"3e4d7d70-68d2-4302-a377-a869f1fd455e", "image": self.image_header, "imageDims": [720,540], "folder_name": self.folder_name, @@ -140,6 +142,7 @@ def test_inference_request_missing_argument(self): expected = ("InferenceRequestError: missing request arguments: either folder_name, container_name, imageDims or image is missing") data = { + "userId":"3e4d7d70-68d2-4302-a377-a869f1fd455e", "image": self.image_header, "imageDims": [720,540], "folder_name": self.folder_name, @@ -189,6 +192,7 @@ def test_inference_request_wrong_pipeline_name(self): "Access-Control-Allow-Origin": "*", }, json={ + "userId":"3e4d7d70-68d2-4302-a377-a869f1fd455e", "image": self.image_src, "imageDims": [720,540], "folder_name": self.folder_name,