revert changes on validate dict

dlt-hub · Nov 21, 2023 · b68013d · b68013d
1 parent 0799197
commit b68013d
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 15 deletions.
diff --git a/dlt/common/schema/utils.py b/dlt/common/schema/utils.py
@@ -246,18 +246,13 @@ def compile_simple_regexes(r: Iterable[TSimpleRegex]) -> REPattern:
 
 
 def validate_stored_schema(stored_schema: TStoredSchema) -> None:
-    # exclude validation of keys added later
-    ignored_keys = []
-    if stored_schema["engine_version"] < 7:
-        ignored_keys.append("previous_hashes")
 
     # use lambda to verify only non extra fields
     validate_dict_ignoring_xkeys(
         spec=TStoredSchema,
         doc=stored_schema,
         path=".",
-        validator_f=simple_regex_validator,
-        filter_required=lambda k: k not in ignored_keys
+        validator_f=simple_regex_validator
     )
     # check child parent relationships
     for table_name, table in stored_schema["tables"].items():

diff --git a/dlt/common/validation.py b/dlt/common/validation.py
@@ -9,7 +9,7 @@
 TCustomValidator = Callable[[str, str, Any, Any], bool]
 
 
-def validate_dict(spec: Type[_TypedDict], doc: StrAny, path: str, filter_f: TFilterFunc = None, validator_f: TCustomValidator = None, filter_required: TFilterFunc = None) -> None:
+def validate_dict(spec: Type[_TypedDict], doc: StrAny, path: str, filter_f: TFilterFunc = None, validator_f: TCustomValidator = None) -> None:
     """Validate the `doc` dictionary based on the given typed dictionary specification `spec`.
 
     Args:
@@ -34,12 +34,11 @@ def validate_dict(spec: Type[_TypedDict], doc: StrAny, path: str, filter_f: TFil
     """
     # pass through filter
     filter_f = filter_f or (lambda _: True)
-    filter_required = filter_required or (lambda _: True)
     # cannot validate anything
     validator_f = validator_f or (lambda p, pk, pv, t: False)
 
     allowed_props = get_type_hints(spec)
-    required_props = {k: v for k, v in allowed_props.items() if (not is_optional_type(v) and filter_required(k))}
+    required_props = {k: v for k, v in allowed_props.items() if not is_optional_type(v)}
     # remove optional props
     props = {k: v for k, v in doc.items() if filter_f(k)}
     # check missing props

diff --git a/tests/common/schema/test_schema.py b/tests/common/schema/test_schema.py
@@ -132,10 +132,10 @@ def test_simple_regex_validator() -> None:
 
 
 def test_load_corrupted_schema() -> None:
-    eth_v4: TStoredSchema = load_yml_case("schemas/eth/ethereum_schema_v8")
-    del eth_v4["tables"]["blocks"]
+    eth_v8: TStoredSchema = load_yml_case("schemas/eth/ethereum_schema_v8")
+    del eth_v8["tables"]["blocks"]
     with pytest.raises(ParentTableNotFoundException):
-        utils.validate_stored_schema(eth_v4)
+        utils.validate_stored_schema(eth_v8)
 
 
 def test_column_name_validator(schema: Schema) -> None:
@@ -287,21 +287,18 @@ def test_upgrade_engine_v1_schema() -> None:
     assert schema_dict["engine_version"] == 2
     upgraded = utils.migrate_schema(schema_dict, from_engine=2, to_engine=4)
     assert upgraded["engine_version"] == 4
-    utils.validate_stored_schema(upgraded)
 
     # upgrade 1 -> 4
     schema_dict = load_json_case("schemas/ev1/event.schema")
     assert schema_dict["engine_version"] == 1
     upgraded = utils.migrate_schema(schema_dict, from_engine=1, to_engine=4)
     assert upgraded["engine_version"] == 4
-    utils.validate_stored_schema(upgraded)
 
     # upgrade 1 -> 6
     schema_dict = load_json_case("schemas/ev1/event.schema")
     assert schema_dict["engine_version"] == 1
     upgraded = utils.migrate_schema(schema_dict, from_engine=1, to_engine=6)
     assert upgraded["engine_version"] == 6
-    utils.validate_stored_schema(upgraded)
 
     # upgrade 1 -> 7
     schema_dict = load_json_case("schemas/ev1/event.schema")