From cb204a09785cdd6b40a16dd485ee0e72a25e1ff9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Werbrouck?= <122839953+Francois-Werbrouck@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:28:29 -0400 Subject: [PATCH] Create an inspection from a json form (#79) * Issue #68: Split Datastore * Issue #68: Fix lint * Issue #77: sql function created * Issue #777 fix schema * Issue #77: Fix function * iSSUE #77 FIX FUNCTION * Issue #77: f fix * Issue #77: Function passing through * Issue #77: test build * issue #77: fix tests * Issue #77: positive test is passing * Issue #77: formated output * Issue #77 Update Secrets * Issue #77: metadata * Issue #77: Updated the inspection Metadata * Issue #68: Formatting module Metadata * Issue #68: tests are passing * Issue #77: change metric_type for enum * Issue #77: updated Architecture * Issue #77: lint * save pipeline_id * fixes * fix tests * Issue #77: Update based on review * Issue #77: Fix link .md * Issue #68: fix file path in tests * rename bin tests * issue #77: Fix test * Issue #77: All tests are passing * Issue #77: update pyproject * Issue #77: doc about saving inspection * issue #77: fix typo doc * Fix #77 review * #77 lint fix --------- Co-authored-by: sylvanie85 --- datastore/FertiScan/__init__.py | 49 --- datastore/__init__.py | 6 +- .../FertiScan/new_inspection_function.sql | 310 ++++++++++++++++++ .../db/bytebase/FertiScan/schema_0.0.7.sql | 69 ++-- .../db/bytebase/FertiScan/schema_0.0.8.sql | 255 ++++++++++++++ .../db/bytebase/FertiScan/schema_0.0.9.sql | 252 ++++++++++++++ datastore/db/metadata/inspection/__init__.py | 302 +++++++++++++++++ datastore/db/queries/inspection/__init__.py | 24 ++ .../db/queries/machine_learning/__init__.py | 6 +- datastore/fertiscan/__init__.py | 71 ++++ datastore/{Nachet => nachet}/__init__.py | 9 +- doc/datastore.md | 4 +- .../fertiScan-architecture.md | 54 +-- doc/fertiscan/new-inspection.md | 206 ++++++++++++ .../deployment-mass-import.md | 0 doc/{Nachet => nachet}/inference-feedback.md | 0 doc/{Nachet => nachet}/inference-results.md | 0 doc/{Nachet => nachet}/nachet-architecture.md | 0 .../nachet-manage-folders.md | 0 doc/{Nachet => nachet}/trusted-user-upload.md | 0 pyproject.toml | 8 +- ..._mass_import.py => BinTest_mass_import.py} | 0 ...e_set.py => BinTest_upload_picture_set.py} | 0 tests/analyse.json | 152 --------- tests/{FertiScan => fertiscan}/__init__.py | 0 tests/fertiscan/analyse.json | 186 +++++++++++ tests/fertiscan/analysis_returned.json | 214 ++++++++++++ .../db/test_inspection.py | 0 .../{FertiScan => fertiscan}/db/test_label.py | 0 .../db/test_metric.py | 0 .../db/test_nutrients.py | 0 .../db/test_organization.py | 0 .../db/test_specification.py | 0 .../db/test_sub_label.py | 0 tests/fertiscan/test_datastore.py | 86 +++++ tests/{Nachet => nachet}/__init__.py | 0 .../db}/inference_example.json | 0 .../db}/ml_structure_exemple.json | 0 tests/{Nachet => nachet}/db/test_inference.py | 41 ++- tests/{Nachet => nachet}/db/test_metadata.py | 11 +- tests/{Nachet => nachet}/db/test_picture.py | 0 tests/{Nachet => nachet}/db/test_seed.py | 0 .../inference_feedback_correction.json | 0 .../inference_feedback_perfect.json | 0 .../{Nachet => nachet}/inference_result.json | 0 tests/nachet/ml_structure_exemple.json | 66 ++++ tests/{Nachet => nachet}/test_datastore.py | 203 +++++++----- tests/test_azure_storage.py | 3 +- tests/test_datastore.py | 45 +-- 49 files changed, 2218 insertions(+), 414 deletions(-) delete mode 100644 datastore/FertiScan/__init__.py create mode 100644 datastore/db/bytebase/FertiScan/new_inspection_function.sql create mode 100644 datastore/db/bytebase/FertiScan/schema_0.0.8.sql create mode 100644 datastore/db/bytebase/FertiScan/schema_0.0.9.sql create mode 100644 datastore/db/metadata/inspection/__init__.py create mode 100644 datastore/fertiscan/__init__.py rename datastore/{Nachet => nachet}/__init__.py (99%) rename doc/{FertiScan => fertiscan}/fertiScan-architecture.md (87%) create mode 100644 doc/fertiscan/new-inspection.md rename doc/{Nachet => nachet}/deployment-mass-import.md (100%) rename doc/{Nachet => nachet}/inference-feedback.md (100%) rename doc/{Nachet => nachet}/inference-results.md (100%) rename doc/{Nachet => nachet}/nachet-architecture.md (100%) rename doc/{Nachet => nachet}/nachet-manage-folders.md (100%) rename doc/{Nachet => nachet}/trusted-user-upload.md (100%) rename tests/{testBin_mass_import.py => BinTest_mass_import.py} (100%) rename tests/{testBin_upload_picture_set.py => BinTest_upload_picture_set.py} (100%) delete mode 100644 tests/analyse.json rename tests/{FertiScan => fertiscan}/__init__.py (100%) create mode 100644 tests/fertiscan/analyse.json create mode 100644 tests/fertiscan/analysis_returned.json rename tests/{FertiScan => fertiscan}/db/test_inspection.py (100%) rename tests/{FertiScan => fertiscan}/db/test_label.py (100%) rename tests/{FertiScan => fertiscan}/db/test_metric.py (100%) rename tests/{FertiScan => fertiscan}/db/test_nutrients.py (100%) rename tests/{FertiScan => fertiscan}/db/test_organization.py (100%) rename tests/{FertiScan => fertiscan}/db/test_specification.py (100%) rename tests/{FertiScan => fertiscan}/db/test_sub_label.py (100%) create mode 100644 tests/fertiscan/test_datastore.py rename tests/{Nachet => nachet}/__init__.py (100%) rename tests/{Nachet => nachet/db}/inference_example.json (100%) rename tests/{Nachet => nachet/db}/ml_structure_exemple.json (100%) rename tests/{Nachet => nachet}/db/test_inference.py (93%) rename tests/{Nachet => nachet}/db/test_metadata.py (95%) rename tests/{Nachet => nachet}/db/test_picture.py (100%) rename tests/{Nachet => nachet}/db/test_seed.py (100%) rename tests/{Nachet => nachet}/inference_feedback_correction.json (100%) rename tests/{Nachet => nachet}/inference_feedback_perfect.json (100%) rename tests/{Nachet => nachet}/inference_result.json (100%) create mode 100644 tests/nachet/ml_structure_exemple.json rename tests/{Nachet => nachet}/test_datastore.py (84%) diff --git a/datastore/FertiScan/__init__.py b/datastore/FertiScan/__init__.py deleted file mode 100644 index 99667ddd..00000000 --- a/datastore/FertiScan/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from dotenv import load_dotenv -import uuid - -load_dotenv() - -FERTISCAN_DB_URL = os.environ.get("FERTISCAN_DB_URL") -if FERTISCAN_DB_URL is None or FERTISCAN_DB_URL == "": - # raise ValueError("FERTISCAN_DB_URL is not set") - print("Warning: FERTISCAN_DB_URL not set") - -FERTISCAN_SCHEMA = os.environ.get("FERTISCAN_SCHEMA") -if FERTISCAN_SCHEMA is None or FERTISCAN_SCHEMA == "": - # raise ValueError("FERTISCAN_SCHEMA is not set") - print("Warning: FERTISCAN_SCHEMA not set") - -FERTISCAN_STORAGE_URL = os.environ.get("FERTISCAN_STORAGE_URL") -if FERTISCAN_STORAGE_URL is None or FERTISCAN_STORAGE_URL == "": - # raise ValueError("FERTISCAN_STORAGE_URL is not set") - print("Warning: FERTISCAN_STORAGE_URL not set") - - -async def register_analysis( - cursor, container_client, analysis_dict, picture_id: str, picture, folder="General" -): - """ - Register an analysis in the database - - Parameters: - - cursor: The cursor object to interact with the database. - - container_client: The container client of the user. - - analysis_dict (dict): The analysis to register in a dict string (soon to be json loaded). - - picture: The picture encoded to upload. - - Returns: - - The analysis_dict with the analysis_id added. - """ - try: - if picture_id is None or picture_id == "": - picture_id = str(uuid.uuid4()) - # if not azure_storage.is_a_folder(container_client, folder): - # azure_storage.create_folder(container_client, folder) - # azure_storage.upload_image(container_client, folder, picture, picture_id) - # analysis_id = analysis.new_analysis(cursor, json.dumps(analysis_dict)) - # analysis_dict["analysis_id"] = str(analysis_id) - return None - except Exception as e: - print(e.__str__()) - raise Exception("Datastore Unhandled Error") diff --git a/datastore/__init__.py b/datastore/__init__.py index 75e3d0f4..765ac069 100644 --- a/datastore/__init__.py +++ b/datastore/__init__.py @@ -113,7 +113,9 @@ async def new_user(cursor, email, connection_string, tier="user") -> User: raise Exception("Datastore Unhandled Error") -async def get_user_container_client(user_id, storage_url, account, key, tier="user"): +async def get_user_container_client( + user_id, storage_url, account, key, tier="user" +): """ Get the container client of a user @@ -308,7 +310,7 @@ async def upload_pictures( if not response: raise BlobUploadError("Error uploading the picture") - picture.update_picture_metadata(cursor, picture_id, json.dumps(data), 0) + picture.update_picture_metadata(cursor, picture_id, json.dumps(data), len(hashed_pictures)) pic_ids.append(picture_id) return pic_ids except BlobUploadError or azure_storage.UploadImageError: diff --git a/datastore/db/bytebase/FertiScan/new_inspection_function.sql b/datastore/db/bytebase/FertiScan/new_inspection_function.sql new file mode 100644 index 00000000..1493737d --- /dev/null +++ b/datastore/db/bytebase/FertiScan/new_inspection_function.sql @@ -0,0 +1,310 @@ +CREATE OR REPLACE FUNCTION "fertiscan_0.0.9".new_inspection(user_id uuid, picture_set_id uuid, input_json jsonb) + RETURNS jsonb + LANGUAGE plpgsql +AS $function$ +DECLARE + label_id uuid; + sub_type_rec RECORD; + fr_values jsonb; + en_values jsonb; + read_language text; + record jsonb; + inspection_id uuid; + company_id uuid; + location_id uuid; + manufacturer_location_id uuid; + manufacturer_id uuid; + read_value text; + read_unit text; + metric_type_id uuid; + value_float float; + unit_id uuid; + specification_id uuid; + sub_label_id uuid; + ingredient_language text; + result_json jsonb := '{}'; +BEGIN + +-- COMPANY + -- Check if company location exists by address + SELECT id INTO location_id + FROM location + WHERE address ILIKE input_json->'company'->>'address' + LIMIT 1; + + IF location_id IS NULL THEN + INSERT INTO location (address) + VALUES ( + input_json->'company'->>'address' + ) + RETURNING id INTO location_id; + END IF; + INSERT INTO organization_information (name,website,phone_number,location_id) + VALUES ( + input_json->'company'->>'name', + input_json->'company'->>'website', + input_json->'company'->>'phone_number', + location_id + ) + RETURNING id INTO company_id; + + -- Update input_json with company_id + input_json := jsonb_set(input_json, '{company,id}', to_jsonb(company_id)); + +-- COMPANY END + +-- MANUFACTURER + -- Check if company location exists by address + SELECT id INTO location_id + FROM location + WHERE address ILIKE input_json->'manufacturer'->>'address' + LIMIT 1; + + IF location_id IS NULL THEN + INSERT INTO location (address) + VALUES ( + input_json->'manufacturer'->>'address' + ) + RETURNING id INTO location_id; + END IF; + INSERT INTO organization_information (name,website,phone_number,location_id) + VALUES ( + input_json->'manufacturer'->>'name', + input_json->'manufacturer'->>'website', + input_json->'manufacturer'->>'phone_number', + location_id + ) + RETURNING id INTO manufacturer_id; + + -- Update input_json with company_id + input_json := jsonb_set(input_json, '{manufacturer,id}', to_jsonb(manufacturer_id)); +-- Manufacturer end + +-- LABEL INFORMATION + INSERT INTO label_information ( + lot_number, npk, registration_number, n, p, k, company_info_id, manufacturer_info_id + ) VALUES ( + input_json->'product'->>'lot_number', + input_json->'product'->>'npk', + input_json->'product'->>'registration_number', + (input_json->'product'->>'n')::float, + (input_json->'product'->>'p')::float, + (input_json->'product'->>'k')::float, + company_id, + manufacturer_id + ) + RETURNING id INTO label_id; + + -- Update input_json with company_id + input_json := jsonb_set(input_json, '{product,id}', to_jsonb(label_id)); + +--LABEL END + +--WEIGHT + -- Loop through each element in the 'weight' array + FOR record IN SELECT * FROM jsonb_array_elements(input_json->'weight') + LOOP + -- Extract the value and unit from the current weight record + read_value := record->>'value'; + read_unit := record->>'unit'; + + -- Convert the weight value to float + value_float := read_value::float; + + -- Check if the weight_unit exists in the unit table + SELECT id INTO unit_id FROM unit WHERE unit = read_unit; + + -- If unit_id is null, the unit does not exist + IF unit_id IS NULL THEN + -- Insert the new unit + INSERT INTO unit (unit, to_si_unit) + VALUES (read_unit, null) -- Adjust to_si_unit value as necessary + RETURNING id INTO unit_id; + END IF; + + -- Insert into metric for weight + INSERT INTO metric (value, unit_id, edited,metric_type_id,label_id) + VALUES (value_float, unit_id, FALSE,'weight'::"fertiscan_0.0.9".metric_type,label_id); + END LOOP; +-- Weight end + +--DENSITY + read_value := input_json -> 'product' -> 'density'->> 'value'; + read_unit := input_json -> 'product' -> 'density'->> 'unit'; + -- Check if density_value is not null and handle density_unit + IF read_value IS NOT NULL THEN + value_float = read_value::float; + -- Check if the density_unit exists in the unit table + SELECT id INTO unit_id FROM unit WHERE unit = read_unit; + + -- If unit_id is null, the unit does not exist + IF unit_id IS NULL THEN + -- Insert the new unit + INSERT INTO unit (unit, to_si_unit) + VALUES (read_unit, null) -- Adjust to_si_unit value as necessary + RETURNING id INTO unit_id; + END IF; + + -- Insert into metric for weight + INSERT INTO metric (value, unit_id, edited,metric_type_id,label_id) + VALUES (value_float, unit_id, FALSE,'density'::"fertiscan_0.0.9".metric_type,label_id); + END IF; +-- DENSITY END + +--VOLUME + read_value := input_json -> 'product' -> 'volume'->> 'value'; + read_unit := input_json -> 'product' -> 'volume'->> 'unit'; + -- Check if density_value is not null and handle density_unit + IF read_value IS NOT NULL THEN + value_float = read_value::float; + -- Check if the density_unit exists in the unit table + SELECT id INTO unit_id FROM unit WHERE unit = read_unit; + + -- If unit_id is null, the unit does not exist + IF unit_id IS NULL THEN + -- Insert the new unit + INSERT INTO unit (unit, to_si_unit) + VALUES (read_unit, null) -- Adjust to_si_unit value as necessary + RETURNING id INTO unit_id; + END IF; + + -- Insert into metric for weight + INSERT INTO metric (value, unit_id, edited,metric_type_id,label_id) + VALUES (value_float, unit_id, FALSE,'volume'::"fertiscan_0.0.9".metric_type,label_id); + END IF; +-- Volume end + + -- SPECIFICATION + FOR ingredient_language IN SELECT * FROM jsonb_object_keys(input_json->'specifications') + LOOP + FOR record IN SELECT * FROM jsonb_array_elements(input_json->'specifications'->ingredient_language) + LOOP + INSERT INTO specification (humidity, ph, solubility, edited,LANGUAGE,label_id) + VALUES ( + (record->>'humidity')::float, + (record->>'ph')::float, + (record->>'solubility')::float, + FALSE, + ingredient_language::"nachet_0.0.9".language, + label_id + ); + END LOOP; + END LOOP; +-- SPECIFICATION END + +-- INGREDIENTS + + -- Loop through each language ('en' and 'fr') + FOR ingredient_language IN SELECT * FROM jsonb_object_keys(input_json->'ingredients') + LOOP + -- Loop through each ingredient in the current language + FOR record IN SELECT * FROM jsonb_array_elements(input_json->'ingredients'->ingredient_language ) + LOOP + -- Extract values from the current ingredient record + read_value := record->> 'value'; + read_unit := record ->> 'unit'; + value_float := read_value::float; + INSERT INTO ingredient (organic, active, name, value, unit, edited, label_id, language) + VALUES ( + Null, -- we cant tell + Null, -- We cant tell + record->>'name', + value_float, + read_unit, + FALSE, -- Assuming edited status + label_id, + ingredient_language::"nachet_0.0.9".language + ); + END LOOP; + END LOOP; +--INGREDIENTS ENDS + +-- SUB LABELS + -- Loop through each sub_type + FOR sub_type_rec IN SELECT id,type_en FROM sub_type + LOOP + -- Extract the French and English arrays for the current sub_type + fr_values := input_json->sub_type_rec.type_en->'fr'; + en_values := input_json->sub_type_rec.type_en->'en'; + -- Ensure both arrays are of the same length + IF jsonb_array_length(fr_values) = jsonb_array_length(en_values) THEN + FOR i IN 0..(jsonb_array_length(fr_values) - 1) + LOOP + INSERT INTO sub_label (text_content_fr,text_content_en, label_id, edited, sub_type_id) + VALUES ( + fr_values->>i, + en_values->>i, + label_id, + FALSE, + sub_type_id + ); + END LOOP; + END IF; + END LOOP; + -- SUB_LABEL END + + -- MICRO NUTRIENTS + -- Extract the French and English arrays for the current + fr_values := input_json->sub_type_rec.type_en->'fr'; + en_values := input_json->sub_type_rec.type_en->'en'; + -- Ensure both arrays are of the same length + --IF jsonb_array_length(fr_values) <> jsonb_array_length(en_values) THEN + -- RAISE EXCEPTION 'French and English micronutrient arrays must be of the same length'; + FOR record IN SELECT * FROM jsonb_array_elements(en_values) + LOOP + INSERT INTO micronutrient (read_name, value, unit, edited, label_id,language) + VALUES ( + record->> 'name', + (record->> 'value')::float, + record->> 'unit', + FALSE, + label_id, + 'en':: public.language + ); + END LOOP; + FOR record IN SELECT * FROM jsonb_array_elements(fr_values) + LOOP + INSERT INTO micronutrient (read_name, value, unit, edited, label_id,language) + VALUES ( + record->> 'name', + (record->> 'value')::float, + record->> 'unit', + FALSE, + label_id, + 'fr'::"nachet_0.0.9".language + ); + END LOOP; +--MICRONUTRIENTS ENDS + +-- GUARANTEED + FOR record IN SELECT * FROM jsonb_array_elements(input_json->'guaranteed_analysis') + LOOP + INSERT INTO micronutrient (read_name, value, unit, edited, label_id) + VALUES ( + record->> 'name', + (record->> 'value')::float, + record->> 'unit', + FALSE, + label_id + ); + END LOOP; +-- GUARANTEED END + +-- INSPECTION + INSERT INTO inspection ( + inspector_id, label_info_id, sample_id, picture_set_id + ) VALUES ( + user_id, -- Assuming inspector_id is handled separately + label_id, + NULL, -- NOT handled yet + picture_set_id -- Assuming picture_set_id is handled separately + ) + RETURNING id INTO inspection_id; + + -- Update input_json with company_id + input_json := jsonb_set(input_json, '{inspection_id}', to_jsonb(inspection_id)); + + RETURN input_json; + +END; +$function$ diff --git a/datastore/db/bytebase/FertiScan/schema_0.0.7.sql b/datastore/db/bytebase/FertiScan/schema_0.0.7.sql index cf0a8c23..75eee958 100644 --- a/datastore/db/bytebase/FertiScan/schema_0.0.7.sql +++ b/datastore/db/bytebase/FertiScan/schema_0.0.7.sql @@ -7,12 +7,11 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - CREATE TABLE "fertiscan_0.0.7"."users" ( + CREATE TABLE "fertiscan_0.0.7"."users" ( "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), "email" text NOT NULL UNIQUE, "registration_date" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updated_at" timestamp, - "default_set_id" uuid REFERENCES "fertiscan_0.0.7".picture_set(id) + "updated_at" timestamp ); CREATE TABLE "fertiscan_0.0.7"."picture_set" ( @@ -23,10 +22,13 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "name" text ); - CREATE TABLE "fertiscan_0.0.7"."picture" ( + alter table "fertiscan_0.0.7".users ADD "default_set_id" uuid REFERENCES "fertiscan_0.0.7".picture_set(id); + + CREATE TABLE "fertiscan_0.0.8"."picture" ( "id" uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY, "picture" json NOT NULL, - "picture_set_id" uuid NOT NULL REFERENCES "fertiscan_0.0.7".picture_set(id), + "nb_obj" int, + "picture_set_id" uuid NOT NULL REFERENCES "fertiscan_0.0.8".picture_set(id), "verified" boolean NOT NULL DEFAULT false, "upload_date" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); @@ -47,16 +49,24 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "name" text, "address" text NOT NULL, "region_id" uuid References "fertiscan_0.0.7".region(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."organization_information" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text NOT NULL, + "website" text, + "phone_number" text, + "location_id" uuid REFERENCES "fertiscan_0.0.7".location(id) ); CREATE TABLE "fertiscan_0.0.7"."organization" ( "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), "name" text NOT NULL, - "website" text, - "phone_number" text, + "information_id" uuid REFERENCES "fertiscan_0.0.7".organization_information(id), "main_location_id" uuid REFERENCES "fertiscan_0.0.7".location(id) ); + Alter table "fertiscan_0.0.7".location ADD "owner_id" uuid REFERENCES "fertiscan_0.0.7".organization(id); CREATE TABLE "fertiscan_0.0.7"."sample" ( @@ -72,13 +82,6 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "to_si_unit" float ); - CREATE TABLE "fertiscan_0.0.7"."metric" ( - "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), - "value" float NOT NULL, - "unit_id" uuid REFERENCES "fertiscan_0.0.7".unit(id), - "edited" boolean - ); - CREATE TABLE "fertiscan_0.0.7"."element_compound" ( "id" SERIAL PRIMARY KEY, "number" int NOT NULL, @@ -95,9 +98,22 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "n" float, "p" float, "k" float, - "weight" uuid REFERENCES "fertiscan_0.0.7".metric(id), - "density" uuid REFERENCES "fertiscan_0.0.7".metric(id), - "volume" uuid REFERENCES "fertiscan_0.0.7".metric(id) + "company_info_id" uuid REFERENCES "fertiscan_0.0.7".organization_information(id), + "manufacturer_info_id" uuid REFERENCES "fertiscan_0.0.7".organization_information(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."metric_type" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "type" text NOT NULL + ); + + CREATE TABLE "fertiscan_0.0.7"."metric" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "value" float NOT NULL, + "edited" boolean, + "unit_id" uuid REFERENCES "fertiscan_0.0.7".unit(id), + "metric_type_id" uuid REFERENCES "fertiscan_0.0.7".metric_type(id), + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id) ); CREATE TABLE "fertiscan_0.0.7"."sub_type" ( @@ -106,14 +122,17 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "type_en" text unique NOT NULL ); - + -- CREATE A TYPE FOR FRENCH/ENGLISH LANGUAGE + CREATE TYPE LANGUAGE AS ENUM ('fr', 'en'); + CREATE TABLE "fertiscan_0.0.7"."specification" ( "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), "humidity" float, "ph" float, "solubility" float, "edited" boolean, - "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id) + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), + "language" LANGUAGE ); CREATE TABLE "fertiscan_0.0.7"."sub_label" ( @@ -132,7 +151,8 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "unit" text NOT NULL, "element_id" int REFERENCES "fertiscan_0.0.7".element_compound(id), "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), - "edited" boolean + "edited" boolean, + "language" LANGUAGE ); CREATE TABLE "fertiscan_0.0.7"."guaranteed" ( @@ -149,8 +169,11 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), "organic" boolean NOT NULL, "name" text NOT NULL, + "value" float, + "unit" text, "edited" boolean, - "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id) + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), + "language" LANGUAGE ); CREATE TABLE "fertiscan_0.0.7"."inspection" ( @@ -161,8 +184,6 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "inspector_id" uuid REFERENCES "fertiscan_0.0.7".users(id), "label_info_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), "sample_id" uuid REFERENCES "fertiscan_0.0.7".sample(id), - "company_id" uuid REFERENCES "fertiscan_0.0.7".organization(id), - "manufacturer_id" uuid REFERENCES "fertiscan_0.0.7".organization(id), "picture_set_id" uuid REFERENCES "fertiscan_0.0.7".picture_set(id) ); @@ -172,7 +193,7 @@ IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'ferti "registration_number" text, "upload_date" timestamp DEFAULT CURRENT_TIMESTAMP, "update_at" timestamp DEFAULT CURRENT_TIMESTAMP, - "latest_inspection_id" uuid REFERENCES "fertiscan_0.0.7".inspection, + "latest_inspection_id" uuid REFERENCES "fertiscan_0.0.7".inspection(id), "owner_id" uuid REFERENCES "fertiscan_0.0.7".organization(id) ); diff --git a/datastore/db/bytebase/FertiScan/schema_0.0.8.sql b/datastore/db/bytebase/FertiScan/schema_0.0.8.sql new file mode 100644 index 00000000..1960c80e --- /dev/null +++ b/datastore/db/bytebase/FertiScan/schema_0.0.8.sql @@ -0,0 +1,255 @@ +--Schema creation "fertiscan_0.0.7" +DO +$do$ +BEGIN +IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'fertiscan_0.0.7')) THEN + + + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + + CREATE TABLE "fertiscan_0.0.7"."users" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "email" text NOT NULL UNIQUE, + "registration_date" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp + ); + + CREATE TABLE "fertiscan_0.0.7"."picture_set" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY, + "picture_set" json NOT NULL, + "owner_id" uuid NOT NULL REFERENCES "fertiscan_0.0.7".users(id), + "upload_date" date NOT NULL DEFAULT current_timestamp, + "name" text + ); + + alter table "fertiscan_0.0.7".users ADD "default_set_id" uuid REFERENCES "fertiscan_0.0.7".picture_set(id); + + CREATE TABLE "fertiscan_0.0.8"."picture" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY, + "picture" json NOT NULL, + "nb_obj" int, + "picture_set_id" uuid NOT NULL REFERENCES "fertiscan_0.0.8".picture_set(id), + "verified" boolean NOT NULL DEFAULT false, + "upload_date" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE "fertiscan_0.0.7"."province" ( + "id" SERIAL PRIMARY KEY, + "name" text UNIQUE NOT NULL + ); + + CREATE TABLE "fertiscan_0.0.7"."region" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "province_id" int REFERENCES "fertiscan_0.0.7".province(id), + "name" text NOT NULL + ); + + CREATE TABLE "fertiscan_0.0.7"."location" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text, + "address" text NOT NULL, + "region_id" uuid References "fertiscan_0.0.7".region(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."organization_information" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text NOT NULL, + "website" text, + "phone_number" text, + "location_id" uuid REFERENCES "fertiscan_0.0.7".location(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."organization" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text NOT NULL, + "information_id" uuid REFERENCES "fertiscan_0.0.7".organization_information(id), + "main_location_id" uuid REFERENCES "fertiscan_0.0.7".location(id) + ); + + + Alter table "fertiscan_0.0.7".location ADD "owner_id" uuid REFERENCES "fertiscan_0.0.7".organization(id); + + CREATE TABLE "fertiscan_0.0.7"."sample" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "number" uuid, + "collection_date" date, + "location" uuid REFERENCES "fertiscan_0.0.7".location(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."unit" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "unit" text NOT NULL, + "to_si_unit" float + ); + + CREATE TABLE "fertiscan_0.0.7"."element_compound" ( + "id" SERIAL PRIMARY KEY, + "number" int NOT NULL, + "name_fr" text NOT NULL, + "name_en" text NOT NULL, + "symbol" text NOT NULL UNIQUE + ); + + CREATE TABLE "fertiscan_0.0.7"."label_information" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "lot_number" text, + "npk" text, + "registration_number" text, + "n" float, + "p" float, + "k" float, + "company_info_id" uuid REFERENCES "fertiscan_0.0.7".organization_information(id), + "manufacturer_info_id" uuid REFERENCES "fertiscan_0.0.7".organization_information(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."metric_type" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "type" text NOT NULL + ); + + CREATE TABLE "fertiscan_0.0.7"."metric" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "value" float NOT NULL, + "edited" boolean, + "unit_id" uuid REFERENCES "fertiscan_0.0.7".unit(id), + "metric_type_id" uuid REFERENCES "fertiscan_0.0.7".metric_type(id), + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."sub_type" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "type_fr" text Unique NOT NULL, + "type_en" text unique NOT NULL + ); + + -- CREATE A TYPE FOR FRENCH/ENGLISH LANGUAGE + CREATE TYPE LANGUAGE AS ENUM ('fr', 'en'); + + CREATE TABLE "fertiscan_0.0.7"."specification" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "humidity" float, + "ph" float, + "solubility" float, + "edited" boolean, + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), + "language" LANGUAGE + ); + + CREATE TABLE "fertiscan_0.0.7"."sub_label" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "text_content_fr" text NOT NULL DEFAULT '', + "text_content_en" text NOT NULL DEFAULT '', + "label_id" uuid NOT NULL REFERENCES "fertiscan_0.0.7"."label_information" ("id"), + "edited" boolean NOT NULL, + "sub_type_id" uuid NOT NULL REFERENCES "fertiscan_0.0.7"."sub_type" ("id") + ); + + CREATE TABLE "fertiscan_0.0.7"."micronutrient" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "read_name" text NOT NULL, + "value" float NOT NULL, + "unit" text NOT NULL, + "element_id" int REFERENCES "fertiscan_0.0.7".element_compound(id), + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), + "edited" boolean, + "language" LANGUAGE + ); + + CREATE TABLE "fertiscan_0.0.7"."guaranteed" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "read_name" text NOT NULL, + "value" float NOT NULL, + "unit" text NOT NULL, + "element_id" int REFERENCES "fertiscan_0.0.7".element_compound(id), + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), + "edited" boolean + ); + + CREATE TABLE "fertiscan_0.0.7"."ingredient" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "organic" boolean, + "active" boolean, + "name" text NOT NULL, + "value" float, + "unit" text, + "edited" boolean, + "label_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), + "language" LANGUAGE + ); + + CREATE TABLE "fertiscan_0.0.7"."inspection" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "verified" boolean DEFAULT false, + "upload_date" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "inspector_id" uuid REFERENCES "fertiscan_0.0.7".users(id), + "label_info_id" uuid REFERENCES "fertiscan_0.0.7".label_information(id), + "sample_id" uuid REFERENCES "fertiscan_0.0.7".sample(id), + "picture_set_id" uuid REFERENCES "fertiscan_0.0.7".picture_set(id) + ); + + CREATE TABLE "fertiscan_0.0.7"."fertilizer" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text UNIQUE NOT NULL, + "registration_number" text, + "upload_date" timestamp DEFAULT CURRENT_TIMESTAMP, + "update_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "latest_inspection_id" uuid REFERENCES "fertiscan_0.0.7".inspection(id), + "owner_id" uuid REFERENCES "fertiscan_0.0.7".organization(id) + ); + + Alter table "fertiscan_0.0.7".inspection ADD "fertilizer_id" uuid REFERENCES "fertiscan_0.0.7".fertilizer(id); + + -- Trigger function for the `user` table + CREATE OR REPLACE FUNCTION update_user_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + -- Trigger for the `user` table + CREATE TRIGGER user_update_before + BEFORE UPDATE ON "fertiscan_0.0.7".users + FOR EACH ROW + EXECUTE FUNCTION update_user_timestamp(); + + -- Trigger function for the `analysis` table + CREATE OR REPLACE FUNCTION update_analysis_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + -- Trigger for the `analysis` table + CREATE TRIGGER analysis_update_before + BEFORE UPDATE ON "fertiscan_0.0.7".inspection + FOR EACH ROW + EXECUTE FUNCTION update_analysis_timestamp(); + + -- Trigger function for the `fertilizer` table + CREATE OR REPLACE FUNCTION update_fertilizer_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.update_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + -- Trigger for the `fertilizer` table + CREATE TRIGGER fertilizer_update_before + BEFORE UPDATE ON "fertiscan_0.0.7".fertilizer + FOR EACH ROW + EXECUTE FUNCTION update_fertilizer_timestamp(); + + -- Insert the default types : [instruction, caution,first_aid, warranty] + INSERT INTO "fertiscan_0.0.7".sub_type(type_fr,type_en) VALUES + ('Instruction','Instruction'), + ('Mise en garde','Caution'), + ('Premier soin','First aid'), + ('Garantie','Warranty'); +END +$do$ diff --git a/datastore/db/bytebase/FertiScan/schema_0.0.9.sql b/datastore/db/bytebase/FertiScan/schema_0.0.9.sql new file mode 100644 index 00000000..7386a9f1 --- /dev/null +++ b/datastore/db/bytebase/FertiScan/schema_0.0.9.sql @@ -0,0 +1,252 @@ +--Schema creation "fertiscan_0.0.9" +DO +$do$ +BEGIN +IF (EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'fertiscan_0.0.9')) THEN + + + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + + CREATE TABLE "fertiscan_0.0.9"."users" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "email" text NOT NULL UNIQUE, + "registration_date" timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp + ); + + CREATE TABLE "fertiscan_0.0.9"."picture_set" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY, + "picture_set" json NOT NULL, + "owner_id" uuid NOT NULL REFERENCES "fertiscan_0.0.9".users(id), + "upload_date" date NOT NULL DEFAULT current_timestamp, + "name" text + ); + + alter table "fertiscan_0.0.9".users ADD "default_set_id" uuid REFERENCES "fertiscan_0.0.9".picture_set(id); + + CREATE TABLE "fertiscan_0.0.9"."picture" ( + "id" uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY, + "picture" json NOT NULL, + "nb_obj" int, + "picture_set_id" uuid NOT NULL REFERENCES "fertiscan_0.0.9".picture_set(id), + "verified" boolean NOT NULL DEFAULT false, + "upload_date" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE "fertiscan_0.0.9"."province" ( + "id" SERIAL PRIMARY KEY, + "name" text UNIQUE NOT NULL + ); + + CREATE TABLE "fertiscan_0.0.9"."region" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "province_id" int REFERENCES "fertiscan_0.0.9".province(id), + "name" text NOT NULL + ); + + CREATE TABLE "fertiscan_0.0.9"."location" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text, + "address" text NOT NULL, + "region_id" uuid References "fertiscan_0.0.9".region(id) + ); + + CREATE TABLE "fertiscan_0.0.9"."organization_information" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text NOT NULL, + "website" text, + "phone_number" text, + "location_id" uuid REFERENCES "fertiscan_0.0.9".location(id) + ); + + CREATE TABLE "fertiscan_0.0.9"."organization" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text NOT NULL, + "information_id" uuid REFERENCES "fertiscan_0.0.9".organization_information(id), + "main_location_id" uuid REFERENCES "fertiscan_0.0.9".location(id) + ); + + + Alter table "fertiscan_0.0.9".location ADD "owner_id" uuid REFERENCES "fertiscan_0.0.9".organization(id); + + CREATE TABLE "fertiscan_0.0.9"."sample" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "number" uuid, + "collection_date" date, + "location" uuid REFERENCES "fertiscan_0.0.9".location(id) + ); + + CREATE TABLE "fertiscan_0.0.9"."unit" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "unit" text NOT NULL, + "to_si_unit" float + ); + + CREATE TABLE "fertiscan_0.0.9"."element_compound" ( + "id" SERIAL PRIMARY KEY, + "number" int NOT NULL, + "name_fr" text NOT NULL, + "name_en" text NOT NULL, + "symbol" text NOT NULL UNIQUE + ); + + CREATE TABLE "fertiscan_0.0.9"."label_information" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "lot_number" text, + "npk" text, + "registration_number" text, + "n" float, + "p" float, + "k" float, + "company_info_id" uuid REFERENCES "fertiscan_0.0.9".organization_information(id), + "manufacturer_info_id" uuid REFERENCES "fertiscan_0.0.9".organization_information(id) + ); + + CREATE TYPE "fertiscan_0.0.9".metric_type as ENUM ('volume', 'weight','density'); + + CREATE TABLE "fertiscan_0.0.9"."metric" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "value" float NOT NULL, + "edited" boolean, + "unit_id" uuid REFERENCES "fertiscan_0.0.9".unit(id), + "metric_type" metric_type, + "label_id" uuid REFERENCES "fertiscan_0.0.9".label_information(id) + ); + + CREATE TABLE "fertiscan_0.0.9"."sub_type" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "type_fr" text Unique NOT NULL, + "type_en" text unique NOT NULL + ); + + -- CREATE A TYPE FOR FRENCH/ENGLISH LANGUAGE + CREATE TYPE "fertiscan_0.0.9".LANGUAGE AS ENUM ('fr', 'en'); + + CREATE TABLE "fertiscan_0.0.9"."specification" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "humidity" float, + "ph" float, + "solubility" float, + "edited" boolean, + "label_id" uuid REFERENCES "fertiscan_0.0.9".label_information(id), + "language" LANGUAGE + ); + + CREATE TABLE "fertiscan_0.0.9"."sub_label" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "text_content_fr" text NOT NULL DEFAULT '', + "text_content_en" text NOT NULL DEFAULT '', + "label_id" uuid NOT NULL REFERENCES "fertiscan_0.0.9"."label_information" ("id"), + "edited" boolean NOT NULL, + "sub_type_id" uuid NOT NULL REFERENCES "fertiscan_0.0.9"."sub_type" ("id") + ); + + CREATE TABLE "fertiscan_0.0.9"."micronutrient" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "read_name" text NOT NULL, + "value" float NOT NULL, + "unit" text NOT NULL, + "element_id" int REFERENCES "fertiscan_0.0.9".element_compound(id), + "label_id" uuid REFERENCES "fertiscan_0.0.9".label_information(id), + "edited" boolean, + "language" LANGUAGE + ); + + CREATE TABLE "fertiscan_0.0.9"."guaranteed" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "read_name" text NOT NULL, + "value" float NOT NULL, + "unit" text NOT NULL, + "element_id" int REFERENCES "fertiscan_0.0.9".element_compound(id), + "label_id" uuid REFERENCES "fertiscan_0.0.9".label_information(id), + "edited" boolean + ); + + CREATE TABLE "fertiscan_0.0.9"."ingredient" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "organic" boolean, + "active" boolean, + "name" text NOT NULL, + "value" float, + "unit" text, + "edited" boolean, + "label_id" uuid REFERENCES "fertiscan_0.0.9".label_information(id), + "language" LANGUAGE + ); + + CREATE TABLE "fertiscan_0.0.9"."inspection" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "verified" boolean DEFAULT false, + "upload_date" timestamp DEFAULT CURRENT_TIMESTAMP, + "updated_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "inspector_id" uuid NOT NULL REFERENCES "fertiscan_0.0.9".users(id), + "label_info_id" uuid REFERENCES "fertiscan_0.0.9".label_information(id), + "sample_id" uuid REFERENCES "fertiscan_0.0.9".sample(id), + "picture_set_id" uuid REFERENCES "fertiscan_0.0.9".picture_set(id) + ); + + CREATE TABLE "fertiscan_0.0.9"."fertilizer" ( + "id" uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + "name" text UNIQUE NOT NULL, + "registration_number" text, + "upload_date" timestamp DEFAULT CURRENT_TIMESTAMP, + "update_at" timestamp DEFAULT CURRENT_TIMESTAMP, + "latest_inspection_id" uuid REFERENCES "fertiscan_0.0.9".inspection(id), + "owner_id" uuid REFERENCES "fertiscan_0.0.9".organization(id) + ); + + Alter table "fertiscan_0.0.9".inspection ADD "fertilizer_id" uuid REFERENCES "fertiscan_0.0.9".fertilizer(id); + + -- Trigger function for the `user` table + CREATE OR REPLACE FUNCTION update_user_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + -- Trigger for the `user` table + CREATE TRIGGER user_update_before + BEFORE UPDATE ON "fertiscan_0.0.9".users + FOR EACH ROW + EXECUTE FUNCTION update_user_timestamp(); + + -- Trigger function for the `analysis` table + CREATE OR REPLACE FUNCTION update_analysis_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + -- Trigger for the `analysis` table + CREATE TRIGGER analysis_update_before + BEFORE UPDATE ON "fertiscan_0.0.9".inspection + FOR EACH ROW + EXECUTE FUNCTION update_analysis_timestamp(); + + -- Trigger function for the `fertilizer` table + CREATE OR REPLACE FUNCTION update_fertilizer_timestamp() + RETURNS TRIGGER AS $$ + BEGIN + NEW.update_at = CURRENT_TIMESTAMP; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + -- Trigger for the `fertilizer` table + CREATE TRIGGER fertilizer_update_before + BEFORE UPDATE ON "fertiscan_0.0.9".fertilizer + FOR EACH ROW + EXECUTE FUNCTION update_fertilizer_timestamp(); + + -- Insert the default types : [instruction, caution,first_aid, warranty] + INSERT INTO "fertiscan_0.0.9".sub_type(type_fr,type_en) VALUES + ('Instruction','Instruction'), + ('Mise en garde','Caution'), + ('Premier soin','First aid'), + ('Garantie','Warranty'); +END +$do$ diff --git a/datastore/db/metadata/inspection/__init__.py b/datastore/db/metadata/inspection/__init__.py new file mode 100644 index 00000000..797a0891 --- /dev/null +++ b/datastore/db/metadata/inspection/__init__.py @@ -0,0 +1,302 @@ +""" +This module contains the function to generate the metadata necessary to interact with the database and the other layers of Fertiscan for all the inspection related objects. +The metadata is generated in a json format and is used to store the metadata in the database. + +""" + +from typing import List +from pydantic import BaseModel, ValidationError +from typing import Optional + +class MissingKeyError(Exception): + pass + +class NPKError(Exception): + pass + +class MetadataFormattingError(Exception): + pass + +class OrganizationInformation(BaseModel): + name: str + address: str + website: str + phone_number: str + +class Value(BaseModel): + value: Optional[float] = None + unit: Optional[str] = None + name: str + +class ValuesObjects(BaseModel): + en: List[Value] + fr: List[Value] + +class SubLabel(BaseModel): + en: List[str] + fr: List[str] + +class Metric(BaseModel): + value: float + unit: str + +class Metrics(BaseModel): + weight: List[Metric] + volume: Metric + density: Metric + +class ProductInformation(BaseModel): + name: str + registration_number: str + lot_number: str + metrics: Metrics + npk: str + warranty: str + n: float + p: float + k: float + verified: bool + + +class Specification(BaseModel): + humidity: float + ph: float + solubility: float + +class Specifications(BaseModel): + en: List[Specification] + fr: List[Specification] + +class Inspection(BaseModel): + company: OrganizationInformation + manufacturer: OrganizationInformation + product: ProductInformation + cautions: SubLabel + instructions: SubLabel + micronutrients: ValuesObjects + ingredients: ValuesObjects + specifications: Specifications + first_aid: SubLabel + guaranteed_analysis: List[Value] + + +def build_inspection_import(analysis_form: dict) -> str: + """ + This funtion build an inspection json object from the pipeline of digitalization analysis. + This serves as the metadata for the inspection object in the database. + + Parameters: + - analysis_form: (dict) The digitalization of the label. + + Returns: + - The inspection db object in a string format. + """ + try: + requiered_keys = [ + "company_name", + "company_address", + "company_website", + "company_phone_number", + "manufacturer_name", + "manufacturer_address", + "manufacturer_website", + "manufacturer_phone_number", + "fertiliser_name", + "registration_number", + "lot_number", + "weight", + "density", + "volume", + "npk", + "warranty", + "cautions_en", + "instructions_en", + "micronutrients_en", + "ingredients_en", + "specifications_en", + "first_aid_en", + "cautions_fr", + "instructions_fr", + "micronutrients_fr", + "ingredients_fr", + "specifications_fr", + "first_aid_fr", + "guaranteed_analysis" + ] + missing_keys = [] + for key in requiered_keys: + if key not in analysis_form: + missing_keys.append(key) + if len(missing_keys) > 0: + raise MissingKeyError(missing_keys) + #data = json.loads(analysis_form) + npk = extract_npk(analysis_form.get("npk")) + company = OrganizationInformation( + name=analysis_form.get("company_name"), + address=analysis_form["company_address"], + website=analysis_form["company_website"], + phone_number=analysis_form["company_phone_number"] + ) + manufacturer = OrganizationInformation( + name=analysis_form["manufacturer_name"], + address=analysis_form["manufacturer_address"], + website=analysis_form["manufacturer_website"], + phone_number=analysis_form["manufacturer_phone_number"] + ) + weights : List[Metric] = [] + for i in range(len(analysis_form["weight"])): + weights.append(Metric(unit=analysis_form["weight"][i]["unit"], value=analysis_form["weight"][i]["value"])) + metrics = Metrics( + weight=weights, + volume=Metric(unit=analysis_form["volume"]["unit"], value=analysis_form["volume"]["value"]), + density=Metric(unit=analysis_form["density"]["unit"], value=analysis_form["density"]["value"]) + ) + product = ProductInformation( + name=analysis_form["fertiliser_name"], + registration_number=analysis_form["registration_number"], + lot_number=analysis_form["lot_number"], + metrics=metrics, + npk=analysis_form["npk"], + warranty=analysis_form["warranty"], + n=npk[0], + p=npk[1], + k=npk[2], + verified=False + ) + + cautions = SubLabel( + en=analysis_form["cautions_en"], + fr=analysis_form["cautions_fr"] + ) + + instructions = SubLabel( + en=analysis_form["instructions_en"], + fr=analysis_form["instructions_fr"] + ) + micro_en: List[Value] = [] + micro_fr: List[Value] = [] + for i in range(len(analysis_form["micronutrients_en"])): + micro_en.append(Value(unit= None if analysis_form["micronutrients_en"][i]["unit"] == "" else analysis_form["micronutrients_en"][i]["unit"], value = None if analysis_form["micronutrients_fr"][i]["value"] == "" else analysis_form["micronutrients_fr"][i]["value"], name=analysis_form["micronutrients_en"][i]["nutrient"])) + for i in range(len(analysis_form["micronutrients_fr"])): + micro_fr.append(Value(unit= None if analysis_form["micronutrients_fr"][i]["unit"] == "" else analysis_form["micronutrients_fr"][i]["unit"], value = None if analysis_form["micronutrients_fr"][i]["value"] == "" else analysis_form["micronutrients_fr"][i]["value"], name=analysis_form["micronutrients_fr"][i]["nutrient"])) + micronutrients = ValuesObjects( + en=micro_en, + fr=micro_fr + ) + ingredients_en : List[Value] = [] + ingredients_fr : List[Value] = [] + for i in range(len(analysis_form["ingredients_en"])): + ingredients_en.append(Value(unit=None if analysis_form["ingredients_en"][i]["unit"] == "" else analysis_form["ingredients_en"][i]["unit"], value=None if analysis_form["ingredients_en"][i]["value"] == "" else analysis_form["ingredients_en"][i]["value"], name=analysis_form["ingredients_en"][i]["nutrient"])) + for i in range(len(analysis_form["ingredients_fr"])): + ingredients_fr.append(Value(unit=None if analysis_form["ingredients_fr"][i]["unit"] == "" else analysis_form["ingredients_fr"][i]["unit"], value=None if analysis_form["ingredients_fr"][i]["value"] == "" else analysis_form["ingredients_fr"][i]["value"], name=analysis_form["ingredients_fr"][i]["nutrient"])) + ingredients = ValuesObjects( + en=ingredients_en, + fr=ingredients_fr + ) + + specifications = Specifications( + en=extract_specifications(analysis_form["specifications_en"]), + fr=extract_specifications(analysis_form["specifications_fr"]) + ) + + first_aid = SubLabel( + en=analysis_form["first_aid_en"], + fr=analysis_form["first_aid_fr"] + ) + + guaranteed : List[Value] = [] + for i in range(len(analysis_form["guaranteed_analysis"])): + guaranteed.append(Value(unit=None if analysis_form["guaranteed_analysis"][i]["unit"] == "" else analysis_form["guaranteed_analysis"][i]["unit"], value=None if analysis_form["guaranteed_analysis"][i]["value"] == "" else analysis_form["guaranteed_analysis"][i]["value"], name=analysis_form["guaranteed_analysis"][i]["nutrient"])) + + + inspection_formatted = Inspection( + company=company, + manufacturer=manufacturer, + product=product, + cautions=cautions, + instructions=instructions, + micronutrients=micronutrients, + ingredients=ingredients, + specifications=specifications, + first_aid=first_aid, + guaranteed_analysis=guaranteed + ) + Inspection(**inspection_formatted.model_dump()) + except MissingKeyError as e: + raise MissingKeyError(f"Missing keys: {e}") + except ValidationError as e: + raise MetadataFormattingError("Error InspectionCreationError not created: "+ str(e)) from None + # print(inspection_formatted.model_dump_json()) + return inspection_formatted.model_dump_json() + + +def split_value_unit(value_unit: str) -> dict: + """ + This function splits the value and unit from a string. + The string must be in the format of "value unit". + + Parameters: + - value_unit: (str) The string to split. + + Returns: + - A dictionary containing the value and unit. + """ + if value_unit is None or value_unit == "" or len(value_unit)<2: + return { + "value": None, + "unit": None + } + # loop through the string and split the value and unit + for i in range(len(value_unit)): + if not (value_unit[i].isnumeric() or value_unit[i] == "." or value_unit[i] == ","): + value = value_unit[:i] + unit = value_unit[i:] + break + # trim the unit of any leading or trailing whitespaces + unit = unit.strip() + if unit == "": + unit = None + return { + "value": value, + "unit": unit + } +def extract_npk(npk:str): + """ + This function extracts the npk values from the string npk. + The string must be in the format of "N-P-K". + + Parameters: + - npk: (str) The string to split. + + Returns: + - A list containing the npk values. + """ + if npk is None or npk == "" or len(npk)<5: + return [None,None,None] + npk_formated=npk.replace("N","-") + npk_formated=npk_formated.replace("P","-") + npk_formated=npk_formated.replace("K","-") + npk_reformated = npk.split("-") + for i in range(len(npk_reformated)): + if not npk_reformated[i].isnumeric(): + NPKError("NPK values must be numeric. Issue with: "+npk_reformated[i] + "in the NPK string: "+npk) + return [float(npk_reformated[0]),float(npk_reformated[1]),float(npk_reformated[2])] + +def extract_specifications(specifications: list) -> list: + """ + This function extracts the specifications from the list of strings. + The strings must be in the format of "value unit". + + Parameters: + - specifications: (list) The list of strings to split. + + Returns: + - A list containing the specifications. + """ + output = [] + if specifications is None or specifications == []: + return output + for specification in specifications: + res = Specification(humidity=specification["humidity"], ph=specification["ph"], solubility=specification["solubility"]) + output.append(res) + return output diff --git a/datastore/db/queries/inspection/__init__.py b/datastore/db/queries/inspection/__init__.py index 83de0591..57b1d4b6 100644 --- a/datastore/db/queries/inspection/__init__.py +++ b/datastore/db/queries/inspection/__init__.py @@ -41,6 +41,30 @@ def new_inspection(cursor, user_id, picture_set_id, verified=False): return cursor.fetchone()[0] except Exception: raise InspectionCreationError("Datastore inspection unhandeled error") + +def new_inspection_with_label_info(cursor, user_id, picture_set_id, label_json): + """ + This function calls the new_inspection function within the database and adds the label information to the inspection. + + Parameters: + - cursor (cursor): The cursor of the database. + - user_id (str): The UUID of the user. + - picture_set_id (str): The UUID of the picture set. + - label_json (str): The label information in a json format. + - verified (boolean, optional): The value if the inspection has been verified by the user. Default is False. + + Returns: + - The json with ids of the inspection and the label information. + """ + try: + query = """ + SELECT new_inspection(%s, %s, %s) + """ + cursor.execute(query, (user_id, picture_set_id, label_json)) + return cursor.fetchone()[0] + except Exception as e: + raise InspectionCreationError(e.__str__()) + def is_inspection_verified(cursor, inspection_id): diff --git a/datastore/db/queries/machine_learning/__init__.py b/datastore/db/queries/machine_learning/__init__.py index 1cb3a938..caf652b4 100644 --- a/datastore/db/queries/machine_learning/__init__.py +++ b/datastore/db/queries/machine_learning/__init__.py @@ -273,8 +273,10 @@ def get_pipeline_id_from_model_name(cursor,model_name:str): ) model_id=cursor.fetchone()[0] return model_id + except(ValueError): + raise NonExistingTaskEWarning(f"Warning: the given model '{model_name}' was not found") except(Exception): - raise PipelineNotFoundError(f"Error: error finding pipelin for model {model_name}") + raise PipelineNotFoundError(f"Error: model not found for model name : {model_name}") def new_model(cursor, model,name,endpoint_name,task_id:int): """ @@ -404,6 +406,8 @@ def get_model_id_from_name(cursor,model_name:str): ) model_id=cursor.fetchone()[0] return model_id + except(ValueError): + raise NonExistingTaskEWarning(f"Warning: the given model '{model_name}' was not found") except(Exception): raise PipelineNotFoundError(f"Error: model not found for model name : {model_name}") diff --git a/datastore/fertiscan/__init__.py b/datastore/fertiscan/__init__.py new file mode 100644 index 00000000..6dd5043f --- /dev/null +++ b/datastore/fertiscan/__init__.py @@ -0,0 +1,71 @@ +import os +from dotenv import load_dotenv +import datastore +import datastore.db.queries.picture as picture +import datastore.db.queries.inspection as inspection +import datastore.db.queries.user as user +import datastore.db.metadata.picture_set as data_picture_set +import datastore.db.metadata.inspection as data_inspection + +load_dotenv() + +FERTISCAN_DB_URL = os.environ.get("FERTISCAN_DB_URL") +if FERTISCAN_DB_URL is None or FERTISCAN_DB_URL == "": + # raise ValueError("FERTISCAN_DB_URL is not set") + print("Warning: FERTISCAN_DB_URL not set") + +FERTISCAN_SCHEMA = os.environ.get("FERTISCAN_SCHEMA") +if FERTISCAN_SCHEMA is None or FERTISCAN_SCHEMA == "": + # raise ValueError("FERTISCAN_SCHEMA is not set") + print("Warning: FERTISCAN_SCHEMA not set") + +FERTISCAN_STORAGE_URL = os.environ.get("FERTISCAN_STORAGE_URL") +if FERTISCAN_STORAGE_URL is None or FERTISCAN_STORAGE_URL == "": + # raise ValueError("FERTISCAN_STORAGE_URL is not set") + print("Warning: FERTISCAN_STORAGE_URL not set") + + +async def register_analysis( + cursor, container_client, user_id, hashed_pictures, analysis_dict, +): + """ + Register an analysis in the database + + Parameters: + - cursor: The cursor object to interact with the database. + - container_client: The container client of the user. + - analysis_dict (dict): The analysis to register in a dict string. + - picture: The picture encoded to upload. + + Returns: + - The analysis_dict with the analysis_id added. + """ + try: + if not user.is_a_user_id(cursor=cursor, user_id=user_id): + raise user.UserNotFoundError( + f"User not found based on the given id: {user_id}" + ) + if not container_client.exists(): + raise datastore.ContainerCreationError( + f"Container not found based on the given user_id: {user_id}" + ) + + #Create picture set for this analysis + picture_set_metadata= data_picture_set.build_picture_set(user_id, len(hashed_pictures)) + picture_set_id = picture.new_picture_set(cursor,picture_set_metadata, user_id,"General") + + #Upload pictures to storage + await datastore.upload_pictures(cursor=cursor,user_id=user_id, container_client=container_client, picture_set_id=picture_set_id, hashed_pictures=hashed_pictures) + + #Register analysis in the database + formatted_analysis = data_inspection.build_inspection_import(analysis_dict) + + analysis_db = inspection.new_inspection_with_label_info(cursor, user_id, picture_set_id, formatted_analysis) + return analysis_db + except inspection.InspectionCreationError: + raise Exception("Datastore Inspection Creation Error") + except data_inspection.MissingKeyError: + raise + except Exception as e: + print(e.__str__()) + raise Exception("Datastore unhandeled error") diff --git a/datastore/Nachet/__init__.py b/datastore/nachet/__init__.py similarity index 99% rename from datastore/Nachet/__init__.py rename to datastore/nachet/__init__.py index 843c4886..be85bbd6 100644 --- a/datastore/Nachet/__init__.py +++ b/datastore/nachet/__init__.py @@ -546,8 +546,7 @@ async def new_perfect_inference_feeback(cursor, inference_id, user_id, boxes_id) except Exception as e: print(e) raise Exception(f"Datastore Unhandled Error : {e}") - - + async def import_ml_structure_from_json_version(cursor, ml_version: dict): """ TODO: build tests @@ -576,7 +575,6 @@ async def import_ml_structure_from_json_version(cursor, ml_version: dict): raise ValueError(f"Model {name_model} not found") machine_learning.new_pipeline(cursor, pipeline_db, pipeline_name, model_ids) - async def get_ml_structure(cursor): """ This function retrieves the machine learning structure from the database. @@ -586,7 +584,7 @@ async def get_ml_structure(cursor): try: ml_structure = {"pipelines": [], "models": []} pipelines = machine_learning.get_active_pipeline(cursor) - if len(pipelines) == 0: + if len(pipelines)==0: raise MLRetrievalError("No Active pipelines found in the database.") model_list = [] for pipeline in pipelines: @@ -624,7 +622,6 @@ async def get_ml_structure(cursor): print(e) raise Exception("Datastore Unhandled Error") - async def get_seed_info(cursor): """ This function retrieves the seed information from the database. @@ -674,7 +671,7 @@ async def delete_picture_set_with_archive( raise picture.PictureSetDeleteError( f"User can't delete the default picture set, user uuid :{user_id}" ) - + folder_name = picture.get_picture_set_name(cursor, picture_set_id) if folder_name is None: folder_name = picture_set_id diff --git a/doc/datastore.md b/doc/datastore.md index eede97f9..c301c652 100644 --- a/doc/datastore.md +++ b/doc/datastore.md @@ -70,8 +70,8 @@ flowchart LR; ## Database Architecture For more detail on each app database architecture go check [Nachet - Architecture](Nachet/nachet-architecture.md) and [Fertiscan - Architecture](FertiScan/fertiScan-architecture.md). + Architecture](nachet/nachet-architecture.md) and [Fertiscan + Architecture](fertiscan/fertiScan-architecture.md). ### Global Needs diff --git a/doc/FertiScan/fertiScan-architecture.md b/doc/fertiscan/fertiScan-architecture.md similarity index 87% rename from doc/FertiScan/fertiScan-architecture.md rename to doc/fertiscan/fertiScan-architecture.md index 69d8bd7f..1bf76810 100644 --- a/doc/FertiScan/fertiScan-architecture.md +++ b/doc/fertiscan/fertiScan-architecture.md @@ -68,10 +68,15 @@ erDiagram } organization{ uuid id PK - string name "unique" + uuid information_id FK + uuid main_location_id FK + } + organization_information{ + uuid id PK + string name string website string phone_number - uuid main_location_id FK + uuid location_id FK } location{ uuid id PK @@ -101,10 +106,8 @@ erDiagram float n float p float k - - uuid weight_id FK - uuid density_id FK - uuid volume_id FK + uuid company_info_id FK + uuid manufacturer_info_id FK } sub_label{ uuid id PK @@ -127,17 +130,6 @@ erDiagram boolean edited uuid label_id FK } - metric{ - uuid id PK - float value - boolean edited - uuid unit_id FK - } - unit{ - uuid id PK - string unit - float to_si_unit - } micronutrient{ uuid id PK string read_name @@ -168,31 +160,43 @@ erDiagram string name_fr string name_en string symbol + } + metric{ + uuid id PK + float value + boolean edited + ENUM metric_type + uuid unit_id FK + uuid label_id FK + } + unit{ + uuid id PK + string unit + float to_si_unit } inspection ||--|| sample :has picture_set ||--|{picture : contains - inspection ||--o| organization: manufacturer - inspection ||--o| organization: company fertilizer ||--|| organization: responsable - location }|--|| organization: host - organization ||--|| location: HQ + organization_information ||--|| location: Hosts location ||--|| region: defines region ||--|| province: apart inspection ||--|| fertilizer : about inspection }|--|| users :inspect inspection ||--o| picture_set :has inspection ||--|| label_information : defines - label_information ||--|o metric: weight - label_information ||--|o metric: density - label_information ||--|o metric: volume label_information ||--|{ ingredient: has label_information ||--|{ guaranteed: has label_information ||--|{ micronutrient: has label_information ||--|{ specification: has label_information ||--|{ sub_label: has + label_information ||--o| organization_information: company + label_information ||--o| organization_information: manufacturer + organization_information ||--|| organization: defines + label_information ||--|{ metric: has sub_label }o--|| sub_type: defines users ||--o{ picture_set: owns - metric ||--|| unit: defines + metric }|--|| unit: defines + micronutrient ||--|| element_compound: is guaranteed ||--|| element_compound: is diff --git a/doc/fertiscan/new-inspection.md b/doc/fertiscan/new-inspection.md new file mode 100644 index 00000000..33d1cb23 --- /dev/null +++ b/doc/fertiscan/new-inspection.md @@ -0,0 +1,206 @@ +# Inspection registration Documentation + +## Context + +The User wants to digitalize a label picture on FertiScan. Therefore, the BE +uses it's models to digitalize the content and sends over a JSON with all the +information taken from the pictures. We need to parse the JSON, saves correctly +the information into the DB linked to the pictures received. + +## Prerequisites + +- The user must be already registered + +## Entity Used + +``` mermaid + +--- +title: FertiScan DB Structure +--- +erDiagram + users{ + uuid id PK + string email + timestamp registration_date + timestamp updated_at + uuid default_set_id FK + } + picture_set{ + uuid id PK + string name + json picture_set + uuid owner_id FK + timestamp upload_date + } + picture{ + uuid id PK + json picture + boolean used_for_digitalization + timestamp upload_date + uuid picture_set_id FK + uuid parent_picture_id FK + } + inspection { + uuid id PK + boolean verified + TIMESTAMP upload_date + TIMESTAMP updated_at + uuid inspector_id FK + uuid label_info_id Fk + uuid fertilizer_id FK + uuid sample_id FK + uuid company_id FK + uuid manufacturer_id FK + uuid picture_set_id FK + } + organization_information{ + uuid id PK + string name + string website + string phone_number + uuid location_id FK + } + location{ + uuid id PK + string address + uuid organization_id FK + uuid region_id FK + } + label_information{ + uuid id PK + string lot_number + string npk + string registration_number + float n + float p + float k + uuid company_info_id FK + uuid manufacturer_info_id FK + } + sub_label{ + uuid id PK + text content_fr + text content_en + boolean edited + uuid label_id FK + uuid sub_type_id FK + } + sub_type{ + id uuid PK + text type_fr "unique" + text type_en "unique" + } + specification{ + id uuid PK + float humidity + float ph + float solubility + boolean edited + uuid label_id FK + } + micronutrient{ + uuid id PK + string read_name + float value + string unit + boolean edited + uuid label_id FK + int element_id FK + } + guaranteed{ + uuid id PK + string read_name + float value + string unit + boolean edited + int element_id FK + uuid label_id FK + } + ingredient{ + uuid id PK + boolean organic + string name + boolean edited + uuid label_id FK + } + element_compound{ + int id PK + string name_fr + string name_en + string symbol + } + metric{ + uuid id PK + float value + boolean edited + ENUM metric_type + uuid unit_id FK + uuid label_id FK + } + unit{ + uuid id PK + string unit + float to_si_unit + } + picture_set ||--|{picture : contains + organization_information ||--|| location: Hosts + inspection }|--|| users :inspect + inspection ||--o| picture_set :has + inspection ||--|| label_information : defines + label_information ||--|{ ingredient: has + label_information ||--|{ guaranteed: has + label_information ||--|{ micronutrient: has + label_information ||--|{ specification: has + label_information ||--|{ sub_label: has + label_information ||--o| organization_information: company + label_information ||--o| organization_information: manufacturer + label_information ||--|{ metric: has + sub_label }o--|| sub_type: defines + users ||--o{ picture_set: owns + metric }|--|| unit: defines + + micronutrient ||--|| element_compound: is + guaranteed ||--|| element_compound: is + +``` + +## Sequence of saving + +```mermaid +sequenceDiagram + title FertiScan Submit Form + actor C as Client + participant FE as Frontend + participant BE as FertiScan + participant DS as DataStore + participant DB as Database + participant blob as BLOB Storage + + C ->> FE: Upload pictures + FE ->> BE: Analysis label (user_id,[pictures]) + BE ->> BE: Digitalize label(pictures) + BE ->> DS: register_analysis(cursor,user_id,pictures,form.json) + + DS ->> DB: new_picture_set(user_id) + activate DS + DB --> DS: picture_set_id + DS ->>blob: new_folder(picture_set_id) + DS ->> DS: upload_pictures(user_id,pictures,picture_set_id,container_client) + DS ->> DB: register all pictures + DB --> DS: picture_ids + DS ->> blob: container_client.upload_pictures(pictures,picture_set_id) + deactivate DS + DS ->> DS: formatted_form = build_inspection_import(form) + DS ->> DB: new_inspection(user_id,picture_set_id,formatted_form.json) + DB --> DS: formatted_form_with_ids.json + + DS --> BE: formatted_form_with_ids.json + BE ->> FE: Display_result(reworked_form_with_ids.json) + FE --> C: Build HTML page based on the received json for confirmation + +``` + +## Sequence of updating + +TODO diff --git a/doc/Nachet/deployment-mass-import.md b/doc/nachet/deployment-mass-import.md similarity index 100% rename from doc/Nachet/deployment-mass-import.md rename to doc/nachet/deployment-mass-import.md diff --git a/doc/Nachet/inference-feedback.md b/doc/nachet/inference-feedback.md similarity index 100% rename from doc/Nachet/inference-feedback.md rename to doc/nachet/inference-feedback.md diff --git a/doc/Nachet/inference-results.md b/doc/nachet/inference-results.md similarity index 100% rename from doc/Nachet/inference-results.md rename to doc/nachet/inference-results.md diff --git a/doc/Nachet/nachet-architecture.md b/doc/nachet/nachet-architecture.md similarity index 100% rename from doc/Nachet/nachet-architecture.md rename to doc/nachet/nachet-architecture.md diff --git a/doc/Nachet/nachet-manage-folders.md b/doc/nachet/nachet-manage-folders.md similarity index 100% rename from doc/Nachet/nachet-manage-folders.md rename to doc/nachet/nachet-manage-folders.md diff --git a/doc/Nachet/trusted-user-upload.md b/doc/nachet/trusted-user-upload.md similarity index 100% rename from doc/Nachet/trusted-user-upload.md rename to doc/nachet/trusted-user-upload.md diff --git a/pyproject.toml b/pyproject.toml index 1869ab5a..fdba2bb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ license = { file = "LICENSE" } keywords = ["Nachet","ailab"] [tool.setuptools] -packages = ["datastore", "datastore.Nachet", "datastore.db", "datastore.bin", +packages = ["datastore", "datastore.nachet", "datastore.db", "datastore.bin", "datastore.db.queries", "datastore.db.queries.picture", "datastore.db.queries.seed", "datastore.db.queries.user", "datastore.db.queries.machine_learning","datastore.db.queries.inference", @@ -42,6 +42,6 @@ dependencies = {file = ["requirements.txt"]} # where = ["ailab"] [project.urls] -"Homepage" = "https://github.com/ai-cfia/nachet-datastore" -"Bug Tracker" = "https://github.com/ai-cfia/nachet-datastore/issues" -Repository = "https://github.com/ai-cfia/nachet-datastore" +"Homepage" = "https://github.com/ai-cfia/ailab-datastore" +"Bug Tracker" = "https://github.com/ai-cfia/ailab-datastore/issues" +Repository = "https://github.com/ai-cfia/ailab-datastore" diff --git a/tests/testBin_mass_import.py b/tests/BinTest_mass_import.py similarity index 100% rename from tests/testBin_mass_import.py rename to tests/BinTest_mass_import.py diff --git a/tests/testBin_upload_picture_set.py b/tests/BinTest_upload_picture_set.py similarity index 100% rename from tests/testBin_upload_picture_set.py rename to tests/BinTest_upload_picture_set.py diff --git a/tests/analyse.json b/tests/analyse.json deleted file mode 100644 index 90cfa29b..00000000 --- a/tests/analyse.json +++ /dev/null @@ -1,152 +0,0 @@ -{ - "company_name": "GreenGrow Fertilizers Inc.", - "company_address": "123 Greenway Blvd, Springfield IL 62701 USA", - "company_website": "www.greengrowfertilizers.com", - "company_phone_number": "+1 800 555 0199", - "manufacturer_name": "AgroTech Industries Ltd.", - "manufacturer_address": "456 Industrial Park Rd, Oakville ON L6H 5V4 Canada", - "manufacturer_website": "www.agrotechindustries.com", - "manufacturer_phone_number": "+1 416 555 0123", - "fertiliser_name": "SuperGrow 20-20-20", - "registration_number": "F12345678", - "lot_number": "L987654321", - "weight_kg": "25", - "weight_lb": "55", - "density": "1.2 g/cm³", - "volume": "20.8 L", - "npk": "20-20-20", - "warranty": "Guaranteed analysis of nutrients.", - "cautions_en": [ - "Keep out of reach of children.", - "Avoid contact with skin and eyes." - ], - "instructions_en": [ - "1. Dissolve 50g in 10L of water.", - "2. Apply every 2 weeks.", - "3. Store in a cool, dry place." - ], - "micronutrients_en": [ - { - "nutrient": "Iron (Fe)", - "value": "0.10", - "unit": "%" - }, - { - "nutrient": "Zinc (Zn)", - "value": "0.05", - "unit": "%" - }, - { - "nutrient": "Manganese (Mn)", - "value": "0.05", - "unit": "%" - } - ], - "organic_ingredients_en": [ - { - "nutrient": "Bone meal", - "value": "5", - "unit": "%" - }, - { - "nutrient": "Seaweed extract", - "value": "3", - "unit": "%" - }, - { - "nutrient": "Humic acid", - "value": "2", - "unit": "%" - } - ], - "inert_ingredients_en": [ - "Clay", - "Sand", - "Perlite" - ], - "specifications_en": [ - { - "humidity": "10", - "ph": "6.5", - "solubility": "100" - } - ], - "first_aid_en": [ - "In case of contact with eyes, rinse immediately with plenty of water and seek medical advice." - ], - "cautions_fr": [ - "Tenir hors de portée des enfants.", - "Éviter le contact avec la peau et les yeux." - ], - "instructions_fr": [ - "1. Dissoudre 50g dans 10L d'eau.", - "2. Appliquer toutes les 2 semaines.", - "3. Conserver dans un endroit frais et sec." - ], - "micronutrients_fr": [ - { - "nutrient": "Fer (Fe)", - "value": "0.10", - "unit": "%" - }, - { - "nutrient": "Zinc (Zn)", - "value": "0.05", - "unit": "%" - }, - { - "nutrient": "Manganèse (Mn)", - "value": "0.05", - "unit": "%" - } - ], - "organic_ingredients_fr": [ - { - "nutrient": "Farine d'os", - "value": "5", - "unit": "%" - }, - { - "nutrient": "Extrait d'algues", - "value": "3", - "unit": "%" - }, - { - "nutrient": "Acide humique", - "value": "2", - "unit": "%" - } - ], - "inert_ingredients_fr": [ - "Argile", - "Sable", - "Perlite" - ], - "specifications_fr": [ - { - "humidity": "10", - "ph": "6.5", - "solubility": "100" - } - ], - "first_aid_fr": [ - "En cas de contact avec les yeux, rincer immédiatement à grande eau et consulter un médecin." - ], - "guaranteed_analysis": [ - { - "nutrient": "Nitrogen (N)", - "value": "20", - "unit": "%" - }, - { - "nutrient": "Phosphate (P2O5)", - "value": "20", - "unit": "%" - }, - { - "nutrient": "Potash (K2O)", - "value": "20", - "unit": "%" - } - ] -} diff --git a/tests/FertiScan/__init__.py b/tests/fertiscan/__init__.py similarity index 100% rename from tests/FertiScan/__init__.py rename to tests/fertiscan/__init__.py diff --git a/tests/fertiscan/analyse.json b/tests/fertiscan/analyse.json new file mode 100644 index 00000000..43d6018e --- /dev/null +++ b/tests/fertiscan/analyse.json @@ -0,0 +1,186 @@ +{ + "company_name":"GreenGrow Fertilizers Inc.", + "company_address":"123 Greenway Blvd, Springfield IL 62701 USA", + "company_website":"www.greengrowfertilizers.com", + "company_phone_number":"+1 800 555 0199", + "manufacturer_name":"AgroTech Industries Ltd.", + "manufacturer_address":"456 Industrial Park Rd, Oakville ON L6H 5V4 Canada", + "manufacturer_website":"www.agrotechindustries.com", + "manufacturer_phone_number":"+1 416 555 0123", + "fertiliser_name":"SuperGrow 20-20-20", + "registration_number":"F12345678", + "lot_number":"L987654321", + "weight":[ + { + "value":"25", + "unit":"kg" + }, + { + "value":"55", + "unit":"lb" + } + ], + "density":{ + "value":"1.2", + "unit":"g/cm³" + }, + "volume":{ + "value":"20.8", + "unit":"L" + }, + "npk":"20-20-20", + "warranty":"Guaranteed analysis of nutrients.", + "cautions_en":[ + "Keep out of reach of children.", + "Avoid contact with skin and eyes." + ], + "instructions_en":[ + "1. Dissolve 50g in 10L of water.", + "2. Apply every 2 weeks.", + "3. Store in a cool, dry place." + ], + "micronutrients_en":[ + { + "nutrient":"Iron (Fe)", + "value":"0.10", + "unit":"%" + }, + { + "nutrient":"Zinc (Zn)", + "value":"0.05", + "unit":"%" + }, + { + "nutrient":"Manganese (Mn)", + "value":"0.05", + "unit":"%" + } + ], + "ingredients_en":[ + { + "nutrient":"Bone meal", + "value":"5", + "unit":"%" + }, + { + "nutrient":"Seaweed extract", + "value":"3", + "unit":"%" + }, + { + "nutrient":"Humic acid", + "value":"2", + "unit":"%" + }, + { + "nutrient":"Clay", + "value":"", + "unit":"" + }, + { + "nutrient":"Sand", + "value":"", + "unit":"" + }, + { + "nutrient":"Perlite", + "value":"", + "unit":"" + } + ], + "specifications_en":[ + { + "humidity":"10", + "ph":"6.5", + "solubility":"100" + } + ], + "first_aid_en":[ + "In case of contact with eyes, rinse immediately with plenty of water and seek medical advice." + ], + "cautions_fr":[ + "Tenir hors de portée des enfants.", + "Éviter le contact avec la peau et les yeux." + ], + "instructions_fr":[ + "1. Dissoudre 50g dans 10L d'eau.", + "2. Appliquer toutes les 2 semaines.", + "3. Conserver dans un endroit frais et sec." + ], + "micronutrients_fr":[ + { + "nutrient":"Fer (Fe)", + "value":"0.10", + "unit":"%" + }, + { + "nutrient":"Zinc (Zn)", + "value":"0.05", + "unit":"%" + }, + { + "nutrient":"Manganèse (Mn)", + "value":"0.05", + "unit":"%" + } + ], + "ingredients_fr":[ + { + "nutrient":"Farine d'os", + "value":"5", + "unit":"%" + }, + { + "nutrient":"Extrait d'algues", + "value":"3", + "unit":"%" + }, + { + "nutrient":"Acide humique", + "value":"2", + "unit":"%" + }, + { + "nutrient":"Argile", + "value":"", + "unit":"" + }, + { + "nutrient":"Sable", + "value":"", + "unit":"" + }, + { + "nutrient":"Perlite", + "value":"", + "unit":"" + } + ], + "specifications_fr":[ + { + "humidity":"10", + "ph":"6.5", + "solubility":"100" + } + ], + "first_aid_fr":[ + "En cas de contact avec les yeux, rincer immédiatement à grande eau et consulter un médecin." + ], + "guaranteed_analysis":[ + { + "nutrient":"Total Nitrogen (N)", + "value":"20", + "unit":"%" + }, + { + "nutrient":"Available Phosphate (P2O5)", + "value":"20", + "unit":"%" + }, + { + "nutrient":"Soluble Potash (K2O)", + "value":"20", + "unit":"%" + } + ] +} diff --git a/tests/fertiscan/analysis_returned.json b/tests/fertiscan/analysis_returned.json new file mode 100644 index 00000000..3836d9ec --- /dev/null +++ b/tests/fertiscan/analysis_returned.json @@ -0,0 +1,214 @@ +{ + "company": { + "id": "66218ae7-0eba-48b4-a06b-b1563bd40f3e", + "name": "GreenGrow Fertilizers Inc.", + "address": "123 Greenway Blvd, Springfield IL 62701 USA", + "website": "www.greengrowfertilizers.com", + "phone_number": "+1 800 555 0199" + }, + "product": { + "k": 20.0, + "n": 20.0, + "p": 20.0, + "id": "0dcab2ce-206a-4763-a7bb-838a30e227da", + "npk": "20-20-20", + "verified": false, + "name": "SuperGrow 20-20-20", + "metrics": { + "volume": { + "unit": "L", + "value": 20.8 + }, + "weight": [ + { + "unit": "kg", + "value": 25.0 + }, + { + "unit": "lb", + "value": 55.0 + } + ], + "density": { + "unit": "g/cm\u00b3", + "value": 1.2 + } + }, + "warranty": "Guaranteed analysis of nutrients.", + "lot_number": "L987654321", + "registration_number": "F12345678" + }, + "cautions": { + "en": [ + "Keep out of reach of children.", + "Avoid contact with skin and eyes." + ], + "fr": [ + "Tenir hors de port\u00e9e des enfants.", + "\u00c9viter le contact avec la peau et les yeux." + ] + }, + "first_aid": { + "en": [ + "In case of contact with eyes, rinse immediately with plenty of water and seek medical advice." + ], + "fr": [ + "En cas de contact avec les yeux, rincer imm\u00e9diatement \u00e0 grande eau et consulter un m\u00e9decin." + ] + }, + "ingredients": { + "en": [ + { + "name": "Bone meal", + "unit": "%", + "value": 5.0 + }, + { + "name": "Seaweed extract", + "unit": "%", + "value": 3.0 + }, + { + "name": "Humic acid", + "unit": "%", + "value": 2.0 + }, + { + "name": "Clay", + "unit": null, + "value": null + }, + { + "name": "Sand", + "unit": null, + "value": null + }, + { + "name": "Perlite", + "unit": null, + "value": null + } + ], + "fr": [ + { + "name": "Farine d'os", + "unit": "%", + "value": 5.0 + }, + { + "name": "Extrait d'algues", + "unit": "%", + "value": 3.0 + }, + { + "name": "Acide humique", + "unit": "%", + "value": 2.0 + }, + { + "name": "Argile", + "unit": null, + "value": null + }, + { + "name": "Sable", + "unit": null, + "value": null + }, + { + "name": "Perlite", + "unit": null, + "value": null + } + ] + }, + "instructions": { + "en": [ + "1. Dissolve 50g in 10L of water.", + "2. Apply every 2 weeks.", + "3. Store in a cool, dry place." + ], + "fr": [ + "1. Dissoudre 50g dans 10L d'eau.", + "2. Appliquer toutes les 2 semaines.", + "3. Conserver dans un endroit frais et sec." + ] + }, + "manufacturer": { + "id": "0f2fe699-3484-4c77-b920-33c3716bcfe3", + "name": "AgroTech Industries Ltd.", + "address": "456 Industrial Park Rd, Oakville ON L6H 5V4 Canada", + "website": "www.agrotechindustries.com", + "phone_number": "+1 416 555 0123" + }, + "inspection_id": "72437b2d-8f1e-4ad2-96f0-8a6b5e77f176", + "micronutrients": { + "en": [ + { + "name": "Iron (Fe)", + "unit": "%", + "value": 0.1 + }, + { + "name": "Zinc (Zn)", + "unit": "%", + "value": 0.05 + }, + { + "name": "Manganese (Mn)", + "unit": "%", + "value": 0.05 + } + ], + "fr": [ + { + "name": "Fer (Fe)", + "unit": "%", + "value": 0.1 + }, + { + "name": "Zinc (Zn)", + "unit": "%", + "value": 0.05 + }, + { + "name": "Mangan\u00e8se (Mn)", + "unit": "%", + "value": 0.05 + } + ] + }, + "specifications": { + "en": [ + { + "ph": 6.5, + "humidity": 10.0, + "solubility": 100.0 + } + ], + "fr": [ + { + "ph": 6.5, + "humidity": 10.0, + "solubility": 100.0 + } + ] + }, + "guaranteed_analysis": [ + { + "name": "Total Nitrogen (N)", + "unit": "%", + "value": 20.0 + }, + { + "name": "Available Phosphate (P2O5)", + "unit": "%", + "value": 20.0 + }, + { + "name": "Soluble Potash (K2O)", + "unit": "%", + "value": 20.0 + } + ] +} diff --git a/tests/FertiScan/db/test_inspection.py b/tests/fertiscan/db/test_inspection.py similarity index 100% rename from tests/FertiScan/db/test_inspection.py rename to tests/fertiscan/db/test_inspection.py diff --git a/tests/FertiScan/db/test_label.py b/tests/fertiscan/db/test_label.py similarity index 100% rename from tests/FertiScan/db/test_label.py rename to tests/fertiscan/db/test_label.py diff --git a/tests/FertiScan/db/test_metric.py b/tests/fertiscan/db/test_metric.py similarity index 100% rename from tests/FertiScan/db/test_metric.py rename to tests/fertiscan/db/test_metric.py diff --git a/tests/FertiScan/db/test_nutrients.py b/tests/fertiscan/db/test_nutrients.py similarity index 100% rename from tests/FertiScan/db/test_nutrients.py rename to tests/fertiscan/db/test_nutrients.py diff --git a/tests/FertiScan/db/test_organization.py b/tests/fertiscan/db/test_organization.py similarity index 100% rename from tests/FertiScan/db/test_organization.py rename to tests/fertiscan/db/test_organization.py diff --git a/tests/FertiScan/db/test_specification.py b/tests/fertiscan/db/test_specification.py similarity index 100% rename from tests/FertiScan/db/test_specification.py rename to tests/fertiscan/db/test_specification.py diff --git a/tests/FertiScan/db/test_sub_label.py b/tests/fertiscan/db/test_sub_label.py similarity index 100% rename from tests/FertiScan/db/test_sub_label.py rename to tests/fertiscan/db/test_sub_label.py diff --git a/tests/fertiscan/test_datastore.py b/tests/fertiscan/test_datastore.py new file mode 100644 index 00000000..da6bbb76 --- /dev/null +++ b/tests/fertiscan/test_datastore.py @@ -0,0 +1,86 @@ +""" +This is a test script for the highest level of the datastore packages. +It tests the functions in the __init__.py files of the datastore packages. +""" +import asyncio +import io +from PIL import Image +import unittest +import json +import datastore.db.__init__ as db +import datastore.__init__ as datastore +import datastore.fertiscan as fertiscan +import datastore.db.metadata.validator as validator +import os + +BLOB_CONNECTION_STRING = os.environ["FERTISCAN_STORAGE_URL"] +if BLOB_CONNECTION_STRING is None or BLOB_CONNECTION_STRING == "": + raise ValueError("FERTISCAN_STORAGE_URL_TESTING is not set") + +DB_CONNECTION_STRING = os.environ.get("FERTISCAN_DB_URL") +if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": + raise ValueError("FERTISCAN_DB_URL is not set") + +DB_SCHEMA = os.environ.get("FERTISCAN_SCHEMA_TESTING") +if DB_SCHEMA is None or DB_SCHEMA == "": + raise ValueError("FERTISCAN_SCHEMA_TESTING is not set") + +BLOB_ACCOUNT = os.environ["FERTISCAN_BLOB_ACCOUNT"] +if BLOB_ACCOUNT is None or BLOB_ACCOUNT == "": + raise ValueError("NACHET_BLOB_ACCOUNT is not set") + +BLOB_KEY = os.environ["FERTISCAN_BLOB_KEY"] +if BLOB_KEY is None or BLOB_KEY == "": + raise ValueError("NACHET_BLOB_KEY is not set") + +class TestDatastore(unittest.IsolatedAsyncioTestCase): + def setUp(self): + + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "analyse.json") + with open(file_path, 'r') as file: + self.analysis_json = json.load(file) + + self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) + self.cursor = self.con.cursor() + db.create_search_path(self.con, self.cursor, DB_SCHEMA) + self.user_email = 'test@email' + self.user_obj= asyncio.run(datastore.new_user(self.cursor,self.user_email,BLOB_CONNECTION_STRING,'test-user')) + + + self.user_id=datastore.User.get_id(self.user_obj) + self.container_client = asyncio.run(datastore.get_user_container_client( + user_id=self.user_id, + storage_url=BLOB_CONNECTION_STRING, + account=BLOB_ACCOUNT, + key=BLOB_KEY, + tier='test-user')) + + self.image = Image.new("RGB", (1980, 1080), "blue") + self.image_byte_array = io.BytesIO() + self.image.save(self.image_byte_array, format="TIFF") + self.pic_encoded = self.image.tobytes() + + + + def tearDown(self): + self.con.rollback() + self.container_client.delete_container() + db.end_query(self.con, self.cursor) + + def test_register_analysis(self): + self.assertTrue(self.container_client.exists()) + analysis = asyncio.run(fertiscan.register_analysis(self.cursor, self.container_client, self.user_id,[self.pic_encoded,self.pic_encoded] ,self.analysis_json)) + self.assertIsNotNone(analysis) + self.assertTrue(validator.is_valid_uuid(analysis["inspection_id"])) + + # print(analysis) + + def test_register_analysis_invalid_user(self): + with self.assertRaises(Exception): + asyncio.run(fertiscan.register_analysis(self.cursor, self.container_client, "invalid_user_id", [self.pic_encoded,self.pic_encoded], self.analysis_json)) + + def test_register_analysy_missing_key(self): + self.analysis_json.pop("specification_en",None) + with self.assertRaises(fertiscan.data_inspection.MissingKeyError): + asyncio.run(fertiscan.register_analysis(self.cursor, self.container_client, self.user_id, [self.pic_encoded,self.pic_encoded], {})) diff --git a/tests/Nachet/__init__.py b/tests/nachet/__init__.py similarity index 100% rename from tests/Nachet/__init__.py rename to tests/nachet/__init__.py diff --git a/tests/Nachet/inference_example.json b/tests/nachet/db/inference_example.json similarity index 100% rename from tests/Nachet/inference_example.json rename to tests/nachet/db/inference_example.json diff --git a/tests/Nachet/ml_structure_exemple.json b/tests/nachet/db/ml_structure_exemple.json similarity index 100% rename from tests/Nachet/ml_structure_exemple.json rename to tests/nachet/db/ml_structure_exemple.json diff --git a/tests/Nachet/db/test_inference.py b/tests/nachet/db/test_inference.py similarity index 93% rename from tests/Nachet/db/test_inference.py rename to tests/nachet/db/test_inference.py index 48808710..40944eb9 100644 --- a/tests/Nachet/db/test_inference.py +++ b/tests/nachet/db/test_inference.py @@ -17,7 +17,7 @@ from datastore.db.metadata import picture as picture_data from datastore.db.metadata import picture_set as picture_set_data from datastore.db.metadata import validator -from datastore.db.queries import inference, picture, seed, user +from datastore.db.queries import inference, picture, seed, user,machine_learning DB_CONNECTION_STRING = os.environ.get("NACHET_DB_URL") if DB_CONNECTION_STRING is None or DB_CONNECTION_STRING == "": @@ -61,12 +61,16 @@ def setUp(self): self.picture_id = picture.new_picture( self.cursor, self.picture, self.picture_set_id, self.seed_id, self.nb_seed ) - with open("tests/Nachet/inference_example.json", "r") as f: + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "inference_example.json") + with open(file_path, "r") as f: self.inference = json.loads(f.read()) self.inference_trim = ( '{"filename": "inference_example", "totalBoxes": 1, "totalBoxes": 1}' ) self.type = 1 + self.model_id = machine_learning.new_model(self.cursor, json.dumps({}), "test_model",'test-endpoint',1) + self.pipeline_id = machine_learning.new_pipeline(self.cursor, json.dumps({}),"test_pipeline",[self.model_id],False) def tearDown(self): self.con.rollback() @@ -77,7 +81,7 @@ def test_new_inference(self): This test checks if the new_inference function returns a valid UUID """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) self.assertTrue( @@ -98,6 +102,7 @@ def test_new_inference_error(self): self.user_id, self.picture_id, self.type, + self.pipeline_id ) def test_new_inference_obj(self): @@ -105,7 +110,7 @@ def test_new_inference_obj(self): This test checks if the new_inference_object function returns a valid UUID """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) for box in self.inference["boxes"]: inference_obj_id = inference.new_inference_object( @@ -121,7 +126,7 @@ def test_new_inference_obj_error(self): This test checks if the new_inference_object function raises an exception when the connection fails """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) mock_cursor = MagicMock() mock_cursor.fetchone.side_effect = Exception("Connection error") @@ -136,7 +141,7 @@ def test_new_seed_object(self): This test checks if the new_seed_object function returns a valid UUID """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) for box in self.inference["boxes"]: inference_obj_id = inference.new_inference_object( @@ -152,7 +157,7 @@ def test_new_seed_object(self): def test_new_seed_object_error(self): inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type @@ -169,7 +174,7 @@ def test_get_inference(self): """ inference_trim = json.loads(self.inference_trim) inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_data = inference.get_inference(self.cursor, str(inference_id)) # inference_json=json.loads(inference_data) @@ -193,7 +198,7 @@ def test_get_inference_object(self): This test checks if the get_inference_object function returns a correctly build object """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type @@ -219,7 +224,7 @@ def test_get_inference_object_error(self): This test checks if the get_inference_object function raise an error if the inference oject does not exist """ inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = "00000000-0000-0000-0000-000000000000" @@ -231,7 +236,7 @@ def test_get_objects_by_inference(self): This test checks if the get_objects_by_inference function returns the corrects objects for an inference """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) total_boxes = len(self.inference["boxes"]) objects_id = [] @@ -263,7 +268,7 @@ def test_get_inference_object_top_id(self): This test checks if the get_inference_object_top_id function returns the correct top_id of an inference object """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type @@ -289,7 +294,7 @@ def test_set_inference_object_verified_id(self): This test checks if the set_inference_object_verified_id function returns a correctly update inference object """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type @@ -323,7 +328,7 @@ def test_set_inference_object_valid(self): This test checks if the set_inference_object_verified_id function returns a correctly update inference object """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type @@ -354,7 +359,7 @@ def test_is_inference_verified(self): Test if is_inference_verified function correctly returns the inference status """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) verified = inference.is_inference_verified(self.cursor, inference_id) @@ -370,7 +375,7 @@ def test_verify_inference_status(self): Test if verify_inference_status function correctly updates the inference status """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type @@ -401,7 +406,7 @@ def test_get_seed_object_id(self): Test if get_seed_object_id function correctly returns the seed object id """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type @@ -427,7 +432,7 @@ def test_get_not_seed_object_id(self): Test if get_seed_object_id function correctly returns the seed object id """ inference_id = inference.new_inference( - self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type + self.cursor, self.inference_trim, self.user_id, self.picture_id, self.type, self.pipeline_id ) inference_obj_id = inference.new_inference_object( self.cursor, inference_id, json.dumps(self.inference["boxes"][0]), self.type diff --git a/tests/Nachet/db/test_metadata.py b/tests/nachet/db/test_metadata.py similarity index 95% rename from tests/Nachet/db/test_metadata.py rename to tests/nachet/db/test_metadata.py index 9bf57fa1..48e4fd0e 100644 --- a/tests/Nachet/db/test_metadata.py +++ b/tests/nachet/db/test_metadata.py @@ -1,3 +1,4 @@ +import os import datastore.db.metadata.inference as inference import datastore.db.metadata.machine_learning as ml_data import json @@ -14,7 +15,11 @@ def setUp(self): self.overlapping = False self.overlapping_indices = 0 self.total_boxes = 1 - with open("tests/Nachet/inference_example.json") as f: + # Get actual path + + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "inference_example.json") + with open(file_path) as f: self.inference_exemple = json.load(f) self.filename = self.inference_exemple["filename"] self.labelOccurrence = self.inference_exemple["labelOccurrence"] @@ -68,7 +73,9 @@ def test_build_object_import(self): class test_machine_learning_functions(unittest.TestCase): def setUp(self): - with open("tests/Nachet/ml_structure_exemple.json", "r") as file: + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "ml_structure_exemple.json") + with open(file_path, "r") as file: self.ml_structure = json.load(file) self.model_name = "that_model_name" self.model_id = "48efe646-5210-4761-908e-a06f95f0c344" diff --git a/tests/Nachet/db/test_picture.py b/tests/nachet/db/test_picture.py similarity index 100% rename from tests/Nachet/db/test_picture.py rename to tests/nachet/db/test_picture.py diff --git a/tests/Nachet/db/test_seed.py b/tests/nachet/db/test_seed.py similarity index 100% rename from tests/Nachet/db/test_seed.py rename to tests/nachet/db/test_seed.py diff --git a/tests/Nachet/inference_feedback_correction.json b/tests/nachet/inference_feedback_correction.json similarity index 100% rename from tests/Nachet/inference_feedback_correction.json rename to tests/nachet/inference_feedback_correction.json diff --git a/tests/Nachet/inference_feedback_perfect.json b/tests/nachet/inference_feedback_perfect.json similarity index 100% rename from tests/Nachet/inference_feedback_perfect.json rename to tests/nachet/inference_feedback_perfect.json diff --git a/tests/Nachet/inference_result.json b/tests/nachet/inference_result.json similarity index 100% rename from tests/Nachet/inference_result.json rename to tests/nachet/inference_result.json diff --git a/tests/nachet/ml_structure_exemple.json b/tests/nachet/ml_structure_exemple.json new file mode 100644 index 00000000..e25b6fd1 --- /dev/null +++ b/tests/nachet/ml_structure_exemple.json @@ -0,0 +1,66 @@ +{ + "version": "0.1.0", + "date": "2024-02-26", + "pipelines": + [ + { + "models": ["that_model_name", "other_model_name"], + "pipeline_name": "First Pipeline", + "created_by": "Avery GoodDataScientist", + "creation_date": "2024-01-01", + "version": 1, + "description": "Pipeline Description", + "job_name": "Job Name", + "dataset": "Dataset Description", + "Accuracy": 0.6908, + "default": true + }, + { + "models": ["that_model_name"], + "pipeline_name": "Second Pipeline", + "created_by": "Avery GoodDataScientist", + "creation_date": "2024-01-01", + "version": 2, + "description": "Pipeline Description", + "job_name": "Job Name", + "dataset": "Dataset Description", + "Accuracy": 0.7989, + "default": true + } + ], + "models": + [ + { + "task": "Classification", + "endpoint": "https://that-model.inference.ml.com/score", + "api_key": "SeCRetKeys", + "content_type": "application/json", + "deployment_platform": "azure", + "endpoint_name": "that-model-endpoint", + "model_name": "that_model_name", + "created_by": "Avery GoodDataScientist", + "creation_date": "2024-01-01", + "version": 5, + "description": "Model Description", + "job_name": "Job Name", + "dataset": "Dataset Description", + "Accuracy": 0.6908 + }, + { + "task": "Object Detection", + "endpoint": "https://other-model.inference.ml.com/score", + "api_key": "SeCRetKeys", + "content_type": "application/json", + "deployment_platform": "aws", + "endpoint_name": "other-model-endpoint", + "model_name": "other_model_name", + "created_by": "Avery GoodDataScientist", + "creation_date": "2023-11-25", + "version": 3, + "description": "Model Description", + "job_name": "Job Name", + "dataset": "Dataset Description", + "Accuracy": 0.9205 + } + ] +} diff --git a/tests/Nachet/test_datastore.py b/tests/nachet/test_datastore.py similarity index 84% rename from tests/Nachet/test_datastore.py rename to tests/nachet/test_datastore.py index 5a1ee72c..f54be4f4 100644 --- a/tests/Nachet/test_datastore.py +++ b/tests/nachet/test_datastore.py @@ -13,7 +13,7 @@ import asyncio import datastore.db.__init__ as db import datastore.__init__ as datastore -import datastore.Nachet.__init__ as Nachet +import datastore.nachet.__init__ as nachet import datastore.db.metadata.validator as validator import datastore.db.queries.seed as seed_query from copy import deepcopy @@ -47,7 +47,9 @@ class test_ml_structure(unittest.TestCase): def setUp(self): - with open("tests/Nachet/ml_structure_exemple.json") as file: + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "ml_structure_exemple.json") + with open(file_path) as file: self.ml_dict = json.load(file) self.con = db.connect_db(DB_CONNECTION_STRING, DB_SCHEMA) self.cursor = self.con.cursor() @@ -62,7 +64,7 @@ def test_import_ml_structure_from_json(self): Test the import function. """ asyncio.run( - Nachet.import_ml_structure_from_json_version(self.cursor, self.ml_dict) + nachet.import_ml_structure_from_json_version(self.cursor, self.ml_dict) ) self.cursor.execute("SELECT id FROM model WHERE name='that_model_name'") model_id = self.cursor.fetchone()[0] @@ -85,7 +87,7 @@ def test_get_ml_structure(self): """ # asyncio.run(datastore.import_ml_structure_from_json_version(self.cursor,self.ml_dict)) - ml_structure = asyncio.run(Nachet.get_ml_structure(self.cursor)) + ml_structure = asyncio.run(nachet.get_ml_structure(self.cursor)) # self.assertDictEqual(ml_structure,self.ml_dict) for pipeline in self.ml_dict["pipelines"]: for key in pipeline: @@ -109,8 +111,8 @@ def test_get_ml_structure_eeror(self): """ mock_cursor = MagicMock() mock_cursor.fetchall.return_value = [] - with self.assertRaises(Nachet.MLRetrievalError): - asyncio.run(Nachet.get_ml_structure(mock_cursor)) + with self.assertRaises(nachet.MLRetrievalError): + asyncio.run(nachet.get_ml_structure(mock_cursor)) class test_picture(unittest.TestCase): @@ -144,7 +146,9 @@ def setUp(self): ) self.seed_name = "test-name" self.seed_id = seed_query.new_seed(self.cursor, self.seed_name) - with open("tests/Nachet/inference_result.json") as file: + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "inference_result.json") + with open(file_path) as file: self.inference = json.load(file) self.folder_name = "test_folder" @@ -158,7 +162,7 @@ def test_upload_picture_unknown(self): Test the upload picture function. """ picture_id = asyncio.run( - Nachet.upload_picture_unknown( + nachet.upload_picture_unknown( self.cursor, self.user_id, self.pic_encoded, self.container_client ) ) @@ -169,14 +173,14 @@ def test_register_inference_result(self): Test the register inference result function. """ picture_id = asyncio.run( - Nachet.upload_picture_unknown( + nachet.upload_picture_unknown( self.cursor, self.user_id, self.pic_encoded, self.container_client ) ) model_id = "test_model_id" result = asyncio.run( - Nachet.register_inference_result( + nachet.register_inference_result( self.cursor, self.user_id, self.inference, picture_id, model_id ) ) @@ -193,7 +197,7 @@ def test_upload_picture_known(self): ) ) picture_id = asyncio.run( - Nachet.upload_picture_known( + nachet.upload_picture_known( self.cursor, self.user_id, self.pic_encoded, @@ -213,9 +217,9 @@ def test_upload_picture_known_error_user_not_found(self): self.cursor, self.container_client, 0, self.user_id ) ) - with self.assertRaises(Nachet.user.UserNotFoundError): + with self.assertRaises(nachet.user.UserNotFoundError): asyncio.run( - Nachet.upload_picture_known( + nachet.upload_picture_known( self.cursor, uuid.uuid4(), self.pic_encoded, @@ -238,7 +242,7 @@ def test_upload_picture_known_connection_error(self): ) with self.assertRaises(Exception): asyncio.run( - Nachet.upload_picture_known( + nachet.upload_picture_known( mock_cursor, self.user_id, self.pic_encoded, @@ -294,7 +298,9 @@ def setUp(self): self.picture_set_id, ) ) - with open("tests/Nachet/inference_result.json") as file: + base_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(base_dir, "inference_result.json") + with open(file_path) as file: self.inference = json.load(file) self.dev_user_id = datastore.user.get_user_id( @@ -319,7 +325,7 @@ def test_find_validated_pictures(self): self.assertEqual( len( asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( self.cursor, str(self.user_id), str(self.picture_set_id) ) ) @@ -333,7 +339,7 @@ def test_find_validated_pictures(self): # Using deepcopy to ensure each inference is a unique object without shared references inference_copy = deepcopy(self.inference) inference = asyncio.run( - Nachet.register_inference_result( + nachet.register_inference_result( self.cursor, self.user_id, inference_copy, @@ -344,7 +350,7 @@ def test_find_validated_pictures(self): inferences.append(inference) asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, inferences[0]["inference_id"], self.user_id, @@ -355,7 +361,7 @@ def test_find_validated_pictures(self): self.assertEqual( len( asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( self.cursor, str(self.user_id), str(self.picture_set_id) ) ) @@ -365,7 +371,7 @@ def test_find_validated_pictures(self): ) asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, inferences[1]["inference_id"], self.user_id, @@ -373,7 +379,7 @@ def test_find_validated_pictures(self): ) ) asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, inferences[2]["inference_id"], self.user_id, @@ -384,7 +390,7 @@ def test_find_validated_pictures(self): self.assertEqual( len( asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( self.cursor, str(self.user_id), str(self.picture_set_id) ) ) @@ -400,7 +406,7 @@ def test_find_validated_pictures_error_user_not_found(self): pictures_id = [] for i in range(3): picture_id = asyncio.run( - Nachet.upload_picture_unknown( + nachet.upload_picture_unknown( self.cursor, self.user_id, self.pic_encoded, @@ -410,9 +416,9 @@ def test_find_validated_pictures_error_user_not_found(self): ) pictures_id.append(picture_id) - with self.assertRaises(Nachet.user.UserNotFoundError): + with self.assertRaises(nachet.user.UserNotFoundError): asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( self.cursor, str(uuid.uuid4()), str(self.picture_set_id) ) ) @@ -425,7 +431,7 @@ def test_find_validated_pictures_error_connection_error(self): mock_cursor.fetchone.side_effect = Exception("Connection error") with self.assertRaises(Exception): asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( mock_cursor, str(self.user_id), str(self.picture_set_id) ) ) @@ -434,9 +440,9 @@ def test_find_validated_pictures_error_picture_set_not_found(self): """ This test checks if the find_validated_pictures function correctly raise an exception if the picture set given doesn't exist in db """ - with self.assertRaises(Nachet.picture.PictureSetNotFoundError): + with self.assertRaises(nachet.picture.PictureSetNotFoundError): asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( self.cursor, str(self.user_id), str(uuid.uuid4()) ) ) @@ -452,9 +458,9 @@ def test_find_validated_pictures_error_not_owner(self): ) not_owner_user_id = datastore.User.get_id(not_owner_user_obj) - with self.assertRaises(Nachet.UserNotOwnerError): + with self.assertRaises(nachet.UserNotOwnerError): asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( self.cursor, str(not_owner_user_id), str(self.picture_set_id) ) ) @@ -480,7 +486,7 @@ def test_delete_picture_set_with_archive(self): # Using deepcopy to ensure each inference is a unique object without shared references inference_copy = deepcopy(self.inference) inference = asyncio.run( - Nachet.register_inference_result( + nachet.register_inference_result( self.cursor, self.user_id, inference_copy, @@ -491,7 +497,7 @@ def test_delete_picture_set_with_archive(self): inferences.append(inference) # Validate 2 of 3 pictures in the picture set asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, inferences[1]["inference_id"], self.user_id, @@ -499,7 +505,7 @@ def test_delete_picture_set_with_archive(self): ) ) asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, inferences[2]["inference_id"], self.user_id, @@ -507,7 +513,7 @@ def test_delete_picture_set_with_archive(self): ) ) validated_pictures = asyncio.run( - Nachet.find_validated_pictures( + nachet.find_validated_pictures( self.cursor, str(self.user_id), str(self.picture_set_id) ) ) @@ -524,7 +530,7 @@ def test_delete_picture_set_with_archive(self): ) dev_picture_set_id = asyncio.run( - Nachet.delete_picture_set_with_archive( + nachet.delete_picture_set_with_archive( self.cursor, str(self.user_id), str(self.picture_set_id), @@ -580,9 +586,9 @@ def test_delete_picture_set_with_archive_error_user_not_found(self): """ This test checks if the delete_picture_set_with_archive function correctly raise an exception if the user given doesn't exist in db """ - with self.assertRaises(Nachet.user.UserNotFoundError): + with self.assertRaises(nachet.user.UserNotFoundError): asyncio.run( - Nachet.delete_picture_set_with_archive( + nachet.delete_picture_set_with_archive( self.cursor, str(uuid.uuid4()), str(self.picture_set_id), @@ -598,7 +604,7 @@ def test_delete_picture_set_with_archive_error_connection_error(self): mock_cursor.fetchone.side_effect = Exception("Connection error") with self.assertRaises(Exception): asyncio.run( - Nachet.delete_picture_set_with_archive( + nachet.delete_picture_set_with_archive( mock_cursor, str(self.user_id), str(self.picture_set_id), @@ -610,9 +616,9 @@ def test_delete_picture_set_with_archive_error_picture_set_not_found(self): """ This test checks if the delete_picture_set_with_archive function correctly raise an exception if the picture set given doesn't exist in db """ - with self.assertRaises(Nachet.picture.PictureSetNotFoundError): + with self.assertRaises(nachet.picture.PictureSetNotFoundError): asyncio.run( - Nachet.delete_picture_set_with_archive( + nachet.delete_picture_set_with_archive( self.cursor, str(self.user_id), str(uuid.uuid4()), @@ -631,9 +637,9 @@ def test_delete_picture_set_with_archive_error_not_owner(self): ) not_owner_user_id = datastore.User.get_id(not_owner_user_obj) - with self.assertRaises(Nachet.UserNotOwnerError): + with self.assertRaises(nachet.UserNotOwnerError): asyncio.run( - Nachet.delete_picture_set_with_archive( + nachet.delete_picture_set_with_archive( self.cursor, str(not_owner_user_id), str(self.picture_set_id), @@ -659,9 +665,9 @@ def test_delete_picture_set_with_archive_error_default_folder(self): general_folder_id = datastore.user.get_default_picture_set( self.cursor, self.user_id ) - with self.assertRaises(Nachet.picture.PictureSetDeleteError): + with self.assertRaises(nachet.picture.PictureSetDeleteError): asyncio.run( - Nachet.delete_picture_set_with_archive( + nachet.delete_picture_set_with_archive( self.cursor, str(self.user_id), str(general_folder_id), @@ -702,31 +708,26 @@ def setUp(self): with open(file_path) as file: self.inference = json.load(file) picture_id = asyncio.run( - Nachet.upload_picture_unknown( + nachet.upload_picture_unknown( self.cursor, self.user_id, self.pic_encoded, self.container_client ) ) model_id = "test_model_id" self.registered_inference = asyncio.run( - Nachet.register_inference_result( + nachet.register_inference_result( self.cursor, self.user_id, self.inference, picture_id, model_id ) ) - # to match frontend field names : - self.registered_inference["inferenceId"] = self.registered_inference["inference_id"] - self.registered_inference.pop("inference_id") - self.registered_inference["userId"] = self.user_id + self.registered_inference["user_id"] = self.user_id self.mock_box = {"topX": 123, "topY": 456, "bottomX": 789, "bottomY": 123} - self.inference_id = self.registered_inference.get("inferenceId") + self.inference_id = self.registered_inference.get("inference_id") self.boxes_id = [] self.top_id = [] - self.unreal_seed_id = Nachet.seed.new_seed(self.cursor, "unreal_seed") + self.unreal_seed_id = nachet.seed.new_seed(self.cursor, "unreal_seed") for box in self.registered_inference["boxes"]: - box["boxId"] = box["box_id"] - box.pop("box_id") - self.boxes_id.append(box["boxId"]) + self.boxes_id.append(box["box_id"]) self.top_id.append(box["top_id"]) - box["classId"] = Nachet.seed.get_seed_id(self.cursor, box["label"]) + box["classId"] = nachet.seed.get_seed_id(self.cursor, box["label"]) def tearDown(self): self.con.rollback() @@ -738,12 +739,12 @@ def test_new_perfect_inference_feedback(self): This test checks if the new_perfect_inference_feeback function correctly updates the inference object after a perfect feedback is given """ asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, self.inference_id, self.user_id, self.boxes_id ) ) for i in range(len(self.boxes_id)): - object = Nachet.inference.get_inference_object( + object = nachet.inference.get_inference_object( self.cursor, self.boxes_id[i] ) # verified_id must be equal to top_id @@ -756,16 +757,16 @@ def test_new_perfect_inference_feedback_error_verified_inference(self): This test checks if the new_perfect_inference_feeback function correctly raise an exception if the inference given is already verified """ asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, self.inference_id, self.user_id, self.boxes_id ) ) self.assertTrue( - Nachet.inference.is_inference_verified(self.cursor, self.inference_id) + nachet.inference.is_inference_verified(self.cursor, self.inference_id) ) - with self.assertRaises(Nachet.inference.InferenceAlreadyVerifiedError): + with self.assertRaises(nachet.inference.InferenceAlreadyVerifiedError): asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, self.inference_id, self.user_id, self.boxes_id ) ) @@ -774,9 +775,9 @@ def test_new_perfect_inference_feedback_error_inference_not_found(self): """ This test checks if the new_perfect_inference_feeback function correctly raise an exception if the inference given doesn't exist in db """ - with self.assertRaises(Nachet.inference.InferenceNotFoundError): + with self.assertRaises(nachet.inference.InferenceNotFoundError): asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, str(uuid.uuid4()), self.user_id, self.boxes_id ) ) @@ -785,9 +786,9 @@ def test_new_perfect_inference_feedback_error_inference_object_not_found(self): """ This test checks if the new_perfect_inference_feeback function correctly raise an exception if one of the inference object given doesn't exist in db """ - with self.assertRaises(Nachet.inference.InferenceObjectNotFoundError): + with self.assertRaises(nachet.inference.InferenceObjectNotFoundError): asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, self.inference_id, self.user_id, @@ -799,9 +800,9 @@ def test_new_perfect_inference_feedback_error_user_not_found(self): """ This test checks if the new_perfect_inference_feeback function correctly raise an exception if the user given doesn't exist in db """ - with self.assertRaises(Nachet.user.UserNotFoundError): + with self.assertRaises(nachet.user.UserNotFoundError): asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( self.cursor, self.inference_id, str(uuid.uuid4()), self.boxes_id ) ) @@ -814,7 +815,7 @@ def test_new_perfect_inference_feedback_connection_error(self): mock_cursor.fetchone.side_effect = Exception("Connection error") with self.assertRaises(Exception): asyncio.run( - Nachet.new_perfect_inference_feeback( + nachet.new_perfect_inference_feeback( mock_cursor, self.inference_id, self.user_id, self.boxes_id ) ) @@ -824,14 +825,18 @@ def test_new_correction_inference_feedback(self): This test checks if the new_correction_inference_feeback function correctly """ self.assertTrue(validator.is_valid_uuid(self.inference_id)) - + #temporary fix until we fix FE + self.registered_inference["inferenceId"]=self.inference_id + self.registered_inference["userId"] = self.user_id + for box in self.registered_inference["boxes"]: + box["boxId"] = box["box_id"] asyncio.run( - Nachet.new_correction_inference_feedback( + nachet.new_correction_inference_feedback( self.cursor, self.registered_inference, 1 ) ) for i in range(len(self.boxes_id)): - object = Nachet.inference.get_inference_object( + object = nachet.inference.get_inference_object( self.cursor, self.boxes_id[i] ) # verified_id must be equal to top_id @@ -847,15 +852,20 @@ def test_new_correction_inference_feedback_new_guess(self): new_top_ids = [] for box in self.registered_inference["boxes"]: box["label"] = box["topN"][1]["label"] - box["classId"] = Nachet.seed.get_seed_id(self.cursor, box["label"]) + box["classId"] = nachet.seed.get_seed_id(self.cursor, box["label"]) new_top_ids.append(box["topN"][1]["object_id"]) + #temporary fix until we fix FE + self.registered_inference["inferenceId"]=self.inference_id + self.registered_inference["userId"] = self.user_id + for box in self.registered_inference["boxes"]: + box["boxId"] = box["box_id"] asyncio.run( - Nachet.new_correction_inference_feedback( + nachet.new_correction_inference_feedback( self.cursor, self.registered_inference, 1 ) ) for i in range(len(self.boxes_id)): - object_db = Nachet.inference.get_inference_object( + object_db = nachet.inference.get_inference_object( self.cursor, self.boxes_id[i] ) # verified_id must be equal to top_id @@ -870,14 +880,19 @@ def test_new_correction_inference_feedback_box_edited(self): self.assertTrue(validator.is_valid_uuid(self.inference_id)) for box in self.registered_inference["boxes"]: box["box"] = self.mock_box + #temporary fix until we fix FE + self.registered_inference["inferenceId"]=self.inference_id + self.registered_inference["userId"] = self.user_id + for box in self.registered_inference["boxes"]: + box["boxId"] = box["box_id"] asyncio.run( - Nachet.new_correction_inference_feedback( + nachet.new_correction_inference_feedback( self.cursor, self.registered_inference, 1 ) ) for box in self.registered_inference["boxes"]: - object_db = Nachet.inference.get_inference_object( - self.cursor, box["boxId"] + object_db = nachet.inference.get_inference_object( + self.cursor, box["box_id"] ) # The new box metadata must be updated self.assertDictEqual(object_db[1], self.mock_box) @@ -894,17 +909,22 @@ def test_new_correction_inference_feedback_not_guess(self): for box in self.registered_inference["boxes"]: box["label"] = "unreal_seed" box["classId"] = self.unreal_seed_id + #temporary fix until we fix FE + self.registered_inference["inferenceId"]=self.inference_id + self.registered_inference["userId"] = self.user_id + for box in self.registered_inference["boxes"]: + box["boxId"] = box["box_id"] asyncio.run( - Nachet.new_correction_inference_feedback( + nachet.new_correction_inference_feedback( self.cursor, self.registered_inference, 1 ) ) for i in range(len(self.boxes_id)): - object_db = Nachet.inference.get_inference_object( + object_db = nachet.inference.get_inference_object( self.cursor, self.boxes_id[i] ) # verified_id must be equal to the new_top_id - new_top_id = Nachet.inference.get_seed_object_id( + new_top_id = nachet.inference.get_seed_object_id( self.cursor, self.unreal_seed_id, object_db[0] ) self.assertTrue(validator.is_valid_uuid(new_top_id)) @@ -920,13 +940,18 @@ def test_new_correction_inference_feedback_not_valid(self): for box in self.registered_inference["boxes"]: box["label"] = "" box["classId"] = "" + #temporary fix until we fix FE + self.registered_inference["inferenceId"]=self.inference_id + self.registered_inference["userId"] = self.user_id + for box in self.registered_inference["boxes"]: + box["boxId"] = box["box_id"] asyncio.run( - Nachet.new_correction_inference_feedback( + nachet.new_correction_inference_feedback( self.cursor, self.registered_inference, 1 ) ) for i in range(len(self.boxes_id)): - object_db = Nachet.inference.get_inference_object( + object_db = nachet.inference.get_inference_object( self.cursor, self.boxes_id[i] ) # verified_id must not be an id @@ -942,19 +967,21 @@ def test_new_correction_inference_feedback_unknown_seed(self): for box in self.registered_inference["boxes"]: box["label"] = "unknown_seed" box["classId"] = "" + #temporary fix until we fix FE + self.registered_inference["inferenceId"]=self.inference_id + self.registered_inference["userId"] = self.user_id + for box in self.registered_inference["boxes"]: + box["boxId"] = box["box_id"] asyncio.run( - Nachet.new_correction_inference_feedback( + nachet.new_correction_inference_feedback( self.cursor, self.registered_inference, 1 ) ) for i in range(len(self.boxes_id)): - object_db = Nachet.inference.get_inference_object( + object_db = nachet.inference.get_inference_object( self.cursor, self.boxes_id[i] ) # verified_id must be equal to an id self.assertTrue(validator.is_valid_uuid(str(object_db[4]))) # valid column must be true self.assertTrue(object_db[6]) - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_azure_storage.py b/tests/test_azure_storage.py index 0949e490..4bbdff28 100644 --- a/tests/test_azure_storage.py +++ b/tests/test_azure_storage.py @@ -62,13 +62,12 @@ def test_mount_nonexisting_container_create(self): tests when a container does not exists and create_container flag is set to True, should create a new container and return the container client """ - not_uuid = "notuuid" + not_uuid = "notauuid" container_client = asyncio.run( mount_container( self.storage_url, not_uuid, True, self.tier, self.credential ) ) - self.assertTrue(container_client.exists()) container_client.delete_container() diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 3e39c4e0..14f6f640 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -128,6 +128,7 @@ def setUp(self): self.user_id=datastore.User.get_id(self.user_obj) self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) self.folder_name = "test_folder" + self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 3, self.user_id, self.folder_name)) def tearDown(self): self.con.rollback() @@ -143,49 +144,14 @@ def test_upload_pictures(self): picture_ids = asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, pictures, self.container_client, picture_set_id)) self.assertTrue(all([validator.is_valid_uuid(picture_id) for picture_id in picture_ids])) self.assertEqual(len(pictures), asyncio.run(datastore.azure_storage.get_image_count(self.container_client, str(picture_set_id)))) + self.assertTrue(len(pictures),(datastore.picture.count_pictures(self.cursor, picture_set_id))) def test_upload_pictures_error_user_not_found(self): """ This test checks if the upload_picture_known function correctly raise an exception if the user given doesn't exist in db """ - pictures = [self.pic_encoded,self.pic_encoded,self.pic_encoded] - picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) with self.assertRaises(datastore.user.UserNotFoundError): - asyncio.run(datastore.upload_pictures(self.cursor, uuid.uuid4(), pictures, self.container_client, picture_set_id)) - - def test_upload_pictures_connection_error(self): - """ - This test checks if the upload_picture_known function correctly raise an exception if the connection to the db fails - """ - mock_cursor = MagicMock() - mock_cursor.fetchone.side_effect = Exception("Connection error") - picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id)) - with self.assertRaises(Exception): - asyncio.run(datastore.upload_pictures(mock_cursor, self.user_id, [self.pic_encoded,self.pic_encoded,self.pic_encoded], self.container_client, picture_set_id)) - -class test_picture_set(unittest.TestCase): - def setUp(self): - self.con = db.connect_db(DB_CONNECTION_STRING,DB_SCHEMA) - self.cursor = self.con.cursor() - db.create_search_path(self.con, self.cursor,DB_SCHEMA) - self.connection_str=BLOB_CONNECTION_STRING - self.user_email="test@email" - self.user_obj= asyncio.run(datastore.new_user(self.cursor,self.user_email,self.connection_str,'test-user')) - self.image = Image.new("RGB", (1980, 1080), "blue") - self.image_byte_array = io.BytesIO() - self.image.save(self.image_byte_array, format="TIFF") - self.pic_encoded = self.image.tobytes() - #self.picture_hash= asyncio.run(azure_storage.generate_hash(self.pic_encoded)) - self.container_name='test-container' - self.user_id=datastore.User.get_id(self.user_obj) - self.container_client = asyncio.run(datastore.get_user_container_client(self.user_id,BLOB_CONNECTION_STRING,BLOB_ACCOUNT,BLOB_KEY,'test-user')) - self.folder_name = "test_folder" - self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name)) - self.pictures_ids = asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, [self.pic_encoded, self.pic_encoded, self.pic_encoded], self.container_client, self.picture_set_id)) - def tearDown(self): - self.con.rollback() - self.container_client.delete_container() - db.end_query(self.con, self.cursor) + asyncio.run(datastore.upload_pictures(self.cursor, str(uuid.uuid4()), self.pic_encoded,self.container_client)) def test_create_picture_set(self): """ @@ -214,10 +180,10 @@ def test_get_picture_sets_info(self) : """ Test the get_picture_sets_info function """ - + picture_ids=asyncio.run(datastore.upload_pictures(self.cursor, self.user_id, [self.pic_encoded,self.pic_encoded,self.pic_encoded], self.container_client, self.picture_set_id)) picture_sets_info = asyncio.run(datastore.get_picture_sets_info(self.cursor, self.user_id)) self.assertEqual(len(picture_sets_info), 2) - self.assertEqual(picture_sets_info.get(str(self.picture_set_id))[1], 3) + self.assertEqual(picture_sets_info.get(str(self.picture_set_id))[1], len(picture_ids)) self.assertEqual(picture_sets_info.get(str(self.picture_set_id))[0], self.folder_name) self.picture_set_id = asyncio.run(datastore.create_picture_set(self.cursor, self.container_client, 0, self.user_id, self.folder_name + "2")) @@ -258,6 +224,7 @@ def test_delete_picture_set_permanently_error_user_not_found(self): """ This test checks if the delete_picture_set_permanently function correctly raise an exception if the user given doesn't exist in db """ + with self.assertRaises(datastore.user.UserNotFoundError): asyncio.run(datastore.delete_picture_set_permanently(self.cursor, str(uuid.uuid4()), str(self.picture_set_id), self.container_client))