Merge branch 'master' into upload-config

.gitignore

@@ -9,3 +9,4 @@ cbc_pg_data
 pg_data
 .vscode
 .idea
+/*.xlsx

connect_ext_ppr/constants.py

@@ -535,3 +535,38 @@
 """
 SUMMARY_TEMPLATE = """Summary:
 {summary}"""
+
+
+CONFIGURATION_SCHEMA_TEMPLATE = """
+{{
+    "type": "object",
+    "properties": {{
+        "hierarchical_files_data": {{
+            "type": "object",
+            "properties": {{
+                "{product_id}": {{
+                    "type": "object",
+                    "properties": {{
+                        "product_level": {{
+                            "type": "object",
+                            "properties": {{
+                                "ResourceCategories": {{
+                                    "type": "object",
+                                    "anyOf": [
+                                        {{"required": ["Name_EN"]}},
+                                        {{"required": ["Name_en"]}}
+                                    ]
+                                }}
+                            }},
+                            "required": ["ResourceCategories"]
+                        }}
+                    }},
+                    "required": ["product_level"]
+                }}
+            }},
+            "required": ["{product_id}"]
+        }}
+    }},
+    "required": ["hierarchical_files_data"]
+}}
+"""

connect_ext_ppr/schemas.py

@@ -6,22 +6,55 @@
 from datetime import datetime
 from typing import Dict, Optional, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, root_validator
 
 from connect_ext_ppr.models.enums import (
     ConfigurationStateChoices,
     DeploymentRequestStatusChoices,
     DeploymentStatusChoices,
     MimeTypeChoices,
     PPRStatusChoices,
+    TasksStatusChoices,
 )
 
 
+_By = Optional[Union[str, Dict[str, str]]]
+Events = Dict[str, Dict[str, Union[datetime, _By]]]
+
+
+def clean_empties_from_dict(data):
+    """
+    Removes inplace all the fields that are None or empty dicts in data.
+    Returns param data, that was modified inplace.
+    If the param is not a dict, will return the param unmodified.
+    :param data: dict
+    :rtype: dict
+    """
+    if not isinstance(data, dict):
+        return data
+
+    for key in list(data.keys()):
+        value = data[key]
+        if isinstance(value, dict):
+            clean_empties_from_dict(value)
+            value = data[key]
+        if not value:
+            del data[key]
+    return data
+
+
 class NonNullSchema(BaseModel):
     def dict(self, *args, **kwargs):
         kwargs['exclude_none'] = True
         return super().dict(*args, **kwargs)
 
+    @root_validator(pre=True)
+    def validate_events(cls, values):
+        events = values.get('events')
+        if events:
+            values['events'] = clean_empties_from_dict(events)
+        return values
+
 
 class VendorSchema(NonNullSchema):
     id: str
@@ -48,7 +81,7 @@ class DeploymentSchema(NonNullSchema):
     owner: VendorSchema
     last_sync_at: datetime
     status: DeploymentStatusChoices
-    events: Dict[str, Dict[str, Union[datetime, str]]]
+    events: Events
 
 
 class FileSchema(NonNullSchema):
@@ -68,7 +101,7 @@ class ConfigurationSchema(NonNullSchema):
     id: str
     file: FileSchema
     state: ConfigurationStateChoices
-    events: Dict[str, Dict[str, Union[datetime, Dict[str, str]]]]
+    events: Events
 
 
 class ConfigurationCreateSchema(NonNullSchema):
@@ -87,7 +120,7 @@ class PPRVersionSchema(NonNullSchema):
     file: FileSchema
     configuration: Optional[ConfigurationReferenceSchema]
     description: Optional[str]
-    events: Dict[str, Dict[str, Union[datetime, str]]]
+    events: Events
     status: PPRStatusChoices
 
 
@@ -114,7 +147,7 @@ class DeploymentRequestSchema(NonNullSchema):
     status: DeploymentRequestStatusChoices
     manually: bool
     delegate_l2: bool
-    events: Dict[str, Dict[str, Union[datetime, str]]]
+    events: Events
 
     class Config:
         orm_mode = True
@@ -146,3 +179,11 @@ class MarketplaceSchema(NonNullSchema):
     external_id: Optional[str]
 
     ppr: Optional[PPRVersionReferenceSchema]
+
+
+class TaskSchema(NonNullSchema):
+    id: str
+    title: str
+    events: Events
+    status: TasksStatusChoices
+    error_message: Optional[str]

connect_ext_ppr/service.py

@@ -14,16 +14,16 @@
 from connect_ext_ppr.models.file import File
 from connect_ext_ppr.models.ppr import PPRVersion
 from connect_ext_ppr.models.replicas import Product
-from connect_ext_ppr.schemas import FileSchema, PPRVersionCreateSchema
+from connect_ext_ppr.schemas import clean_empties_from_dict, FileSchema, PPRVersionCreateSchema
 from connect_ext_ppr.utils import (
     build_summary,
-    clean_empties_from_dict,
     create_ppr_to_media,
     get_base_workbook,
     get_configuration_from_media,
     get_ppr_from_media,
     get_product_items,
     process_ppr,
+    validate_configuration_schema,
     validate_ppr_schema,
     workbook_to_dict,
 )
@@ -213,11 +213,11 @@ def create_ppr(ppr, context, deployment, db, client, logger):
             ws.to_excel(writer, ws.name, index=False)
 
         file_obj = open(file.name, 'rb')
+        writer.book.save(file_obj.name)
+
         file_obj.seek(0, os.SEEK_END)
         file_size = file_obj.tell()
         file_obj.seek(0)
-        writer.book.save(file_obj.name)
-
         file_name = PPR_FILE_NAME.format(
             product_id=deployment.product_id,
             version=new_version,
@@ -286,3 +286,8 @@ def create_ppr(ppr, context, deployment, db, client, logger):
         logger.error(ex)
         db.rollback()
         raise ExtensionHttpError.EXT_003()
+
+
+def validate_configuration(client, deployment, file_data):
+    data = get_configuration_from_media(client, deployment.account_id, deployment.id, file_data.id)
+    return validate_configuration_schema(data, deployment.product_id)

connect_ext_ppr/utils.py

@@ -15,7 +15,12 @@
 import jwt
 import pandas as pd
 
-from connect_ext_ppr.constants import BASE_SCHEMA, PPR_SCHEMA, SUMMARY_TEMPLATE
+from connect_ext_ppr.constants import (
+    BASE_SCHEMA,
+    CONFIGURATION_SCHEMA_TEMPLATE,
+    PPR_SCHEMA,
+    SUMMARY_TEMPLATE,
+)
 from connect_ext_ppr.errors import ExtensionHttpError
 from connect_ext_ppr.models.enums import MimeTypeChoices
 from connect_ext_ppr.models.deployment import Deployment
@@ -31,30 +36,10 @@
     PPRVersionReferenceSchema,
     PPRVersionSchema,
     ProductSchema,
+    TaskSchema,
 )
 
 
-def clean_empties_from_dict(data):
-    """
-    Removes inplace all the fields that are None or empty dicts in data.
-    Returns param data, that was modified inplace.
-    If the param is not a dict, will return the param unmodified.
-    :param data: dict
-    :rtype: dict
-    """
-    if not isinstance(data, dict):
-        return data
-
-    for key in list(data.keys()):
-        value = data[key]
-        if isinstance(value, dict):
-            clean_empties_from_dict(value)
-            value = data[key]
-        if not value:
-            del data[key]
-    return data
-
-
 class FileColletion:
     PPR = 'pprs'
     CONFIFURATION = 'configurations'
@@ -251,7 +236,7 @@ def get_deployment_request_schema(deployment_request, hub):
         id=ppr.id,
         version=ppr.version,
     )
-    events = clean_empties_from_dict({
+    events = {
         'created': {
             'at': deployment_request.created_at,
             'by': deployment_request.created_by,
@@ -262,7 +247,7 @@ def get_deployment_request_schema(deployment_request, hub):
             'at': deployment_request.aborted_at,
             'by': deployment_request.aborted_by,
         },
-    })
+    }
 
     return DeploymentRequestSchema(
         id=deployment_request.id,
@@ -275,6 +260,32 @@ def get_deployment_request_schema(deployment_request, hub):
     )
 
 
+def get_task_schema(task):
+    return TaskSchema(
+        id=task.id,
+        title=task.title,
+        events={
+            'created': {
+                'at': task.created_at,
+                'by': task.created_by,
+            },
+            'started': {
+                'at': task.started_at,
+            },
+            'finished': {
+                'at': task.finished_at,
+            },
+            'aborted': {
+                'at': task.aborted_at,
+                'by': task.aborted_by,
+            },
+
+        },
+        status=task.status,
+        error_message=task.error_message,
+    )
+
+
 def get_client_object(client, collection_name, obj_id):
     """
     Get client object by id
@@ -433,6 +444,15 @@ def validate_ppr_schema(dict_file: Dict[str, Any]):
         return _parse_json_schema_error(ex)
 
 
+def validate_configuration_schema(dict_file: Dict[str, Any], product_id):
+    schema_string = CONFIGURATION_SCHEMA_TEMPLATE.format(product_id=product_id)
+    schema = json.loads(schema_string)
+    try:
+        jsonschema.validate(dict_file, schema)
+    except jsonschema.ValidationError as ex:
+        return _parse_json_schema_error(ex)
+
+
 def get_base_workbook(data: Optional[bytes]):
     file = NamedTemporaryFile(suffix='.xlsx')
     writer = pd.ExcelWriter(file.name, engine='openpyxl')