🌳 Fix output parsing

core/harambe_core/parser/expression/functions.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from slugify import slugify as python_slugify
+# from slugify import slugify as python_slugify
 
 from harambe_core.parser.expression.evaluator import ExpressionEvaluator
 

core/harambe_core/parser/parser.py

@@ -51,10 +51,6 @@ def validate(self, data: dict[str, Any], base_url: str) -> dict[str, Any]:
         self.field_types = self._get_field_types(base_url)
         model = self._schema_to_pydantic_model(self.schema)
 
-        if self._all_fields_empty(data):
-            raise SchemaValidationError(
-                message="All fields are null or empty.",
-            )
         try:
             res = model(**data).model_dump()
             if self._pk_expression:
@@ -245,25 +241,6 @@ def _get_type(self, field: SchemaFieldType, required: bool | None) -> Type[Any]:
             field_type = Optional[field_type]
         return field_type
 
-    def _all_fields_empty(self, data: dict[str, Any]) -> bool:
-        """
-        Recursively check if all fields in the data are either None or empty.
-        This includes handling nested dictionaries and lists.
-        """
-
-        def is_empty(value: Any) -> bool:
-            if value is None:
-                return True
-            if isinstance(value, dict):
-                return all(is_empty(v) for v in value.values())
-            if isinstance(value, list):
-                return all(is_empty(v) for v in value)
-            if isinstance(value, str):
-                return not value.strip()
-            return False
-
-        return all(is_empty(value) for value in data.values())
-
 
 def base_model_factory(
     config: ConfigDict, computed_fields: dict[str, str], evaluator: ExpressionEvaluator
@@ -304,6 +281,32 @@ def evaluate_computed_fields(self) -> Self:
             for field, expression in computed_fields.items():
                 res = evaluator.evaluate(expression, self)
                 setattr(self, field, res)
+
+            if _all_fields_empty(self.model_dump()):
+                raise SchemaValidationError(
+                    message="All fields are null or empty.",
+                )
+
             return self
 
     return PreValidatedBaseModel
+
+
+def _all_fields_empty(data: dict[str, Any]) -> bool:
+    """
+    Recursively check if all fields in the data are either None or empty.
+    This includes handling nested dictionaries and lists.
+    """
+
+    def is_empty(value: Any) -> bool:
+        if value is None:
+            return True
+        if isinstance(value, dict):
+            return all(is_empty(v) for v in value.values())
+        if isinstance(value, list):
+            return all(is_empty(v) for v in value)
+        if isinstance(value, str):
+            return not value.strip()
+        return False
+
+    return all(is_empty(value) for value in data.values())

core/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "harambe-core"
-version = "0.50.0"
+version = "0.50.1"
 description = "Core types for harambe SDK 🐒🍌"
 authors = [
     { name = "Adam Watkins", email = "[email protected]" }

core/test/parser/test_null_values.py

@@ -118,7 +118,7 @@ def test_pydantic_schema_validation_error_fail(data: Dict[str, Any]) -> None:
                     "code_type": "",
                     "code": "",
                     "code_description": "",
-                    "description": "",
+                    "description": "Somthing",
                 }
             ],
         },
@@ -127,3 +127,59 @@ def test_pydantic_schema_validation_error_fail(data: Dict[str, Any]) -> None:
 def test_pydantic_schema_validation_success(data: Dict[str, Any]):
     validator = SchemaParser(government_contracts)
     validator.validate(data, base_url="http://example.com")
+
+
+@pytest.mark.parametrize(
+    "data",
+    [
+        {"group": "Team", "members": [{}]},
+        {
+            "group": "Team",
+            "members": [
+                {
+                    "name": "",
+                    "age": None,
+                },
+            ],
+        },
+    ],
+)
+def test_with_emtpy_objects(data):
+    schema = {
+        "group": {"type": "string"},
+        "members": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "age": {"type": "integer"},
+                },
+            },
+        },
+    }
+
+    with pytest.raises(SchemaValidationError):
+        validator = SchemaParser(schema)
+        validator.validate(data, base_url="http://example.com")
+
+
+@pytest.mark.parametrize(
+    "strings",
+    [
+        (
+            ["", None, None],
+            [None, None],
+            ["a", "  "],
+            ["a", "b", "c", ""],
+        )
+    ],
+)
+def test_with_empty_literals(strings):
+    schema = {
+        "strings": {"type": "array", "items": {"type": "integer"}},
+    }
+
+    with pytest.raises(SchemaValidationError):
+        validator = SchemaParser(schema)
+        validator.validate({"strings": strings}, base_url="http://example.com")

core/test/parser/test_parser.py

@@ -37,8 +37,8 @@
         (
             load_schema("contact"),
             {
-                "name": {"first_name": None, "last_name": None},
-                "address": {"street": None, "city": None, "zip": None},
+                "name": {"first_name": "Adam", "last_name": None},
+                "address": {"street": None, "city": None, "zip": "9104"},
                 "phone_numbers": [{"type": "mobile", "number": "+1 (628) 555-3456"}],
             },
         ),

sdk/pyproject.toml

@@ -1,14 +1,14 @@
 [project]
 name = "harambe-sdk"
-version = "0.50.0"
+version = "0.50.1"
 description = "Data extraction SDK for Playwright 🐒🍌"
 authors = [
     { name = "Adam Watkins", email = "[email protected]" }
 ]
 requires-python = ">=3.11,<4.0"
 readme = "README.md"
 dependencies = [
-    "harambe_core==0.50.0",
+    "harambe_core==0.50.1",
     "pydantic==2.9.2",
     "playwright==1.47.0",
     "setuptools==73.0.0",

sdk/test/test_e2e.py

@@ -249,10 +249,10 @@ async def scraper(sdk: SDK, *args, **kwargs):
     assert observer.data[0]["page_content"] == observer.data[1]["table_content"]
     for text in ["Apple", "Orange", "Banana"]:
         assert (
-            text in observer.data[0]["page_content"]
+                text in observer.data[0]["page_content"]
         ), f"{text} not in {observer.data[0]['page_content']}"
         assert (
-            text in observer.data[1]["table_content"]
+                text in observer.data[1]["table_content"]
         ), f"{text} not in {observer.data[1]['table_content']}"
 
 
@@ -361,13 +361,13 @@ async def scraper(sdk: SDK, *args, **kwargs):
         ({"key1": "value1", "key2": 2}, '{"key1": "value1", "key2": 2}'),
         # Nested structure
         (
-            {"list": [1, 2, {"nested": "value"}]},
-            '{"list": [1, 2, {"nested": "value"}]}',
+                {"list": [1, 2, {"nested": "value"}]},
+                '{"list": [1, 2, {"nested": "value"}]}',
         ),
     ],
 )
 async def test_load_local_storage(
-    server, observer, harness, test_value, expected_value
+        server, observer, harness, test_value, expected_value
 ):
     local_storage_entry_1 = {
         "domain": "asim-shrestha.com",
@@ -541,8 +541,8 @@ async def scrape(sdk: SDK, url, context) -> None:
     assert observer.data[0]["solicitation_id"] == "6100062375"
     assert observer.data[0]["title"] == "23SW SGL 111 Conn Road"
     assert (
-        observer.data[0]["description"]
-        == "The State of Pennsylvania is seeking proposals for IT services"
+            observer.data[0]["description"]
+            == "The State of Pennsylvania is seeking proposals for IT services"
     )
     assert observer.data[0]["status"] == "Open"
     assert len(observer.data[0]["attachments"]) == 4