Add container_type field to TColumnSchema

dlt/common/libs/pydantic.py

@@ -124,6 +124,9 @@ def pydantic_to_table_schema_columns(
             # try to coerce unknown type to text
             data_type = "text"
 
+        # This is when we have List[BaseModel]
+        # We need to access first argument
+        # to extract type hint to extract schema
         if data_type == "complex" and inner_type == list:
             args = get_args(annotation)
             # If it is just a generic List annotation
@@ -140,9 +143,12 @@ def pydantic_to_table_schema_columns(
                     "nullable": nullable,
                     "container_type": list
                 }
+                continue
 
         # if not is_inner_type_pydantic_model and data_type == "complex" and skip_complex_types:
         #     continue
+
+        # This case is for a single field schema/model
         elif is_inner_type_pydantic_model:
             schema_hints = pydantic_to_table_schema_columns(field.annotation)
 

dlt/common/schema/typing.py

@@ -7,6 +7,7 @@
     Optional,
     Sequence,
     Set,
+    Tuple,
     Type,
     TypedDict,
     NewType,
@@ -111,6 +112,7 @@ class TColumnSchema(TColumnSchemaBase, total=False):
     root_key: Optional[bool]
     merge_key: Optional[bool]
     variant: Optional[bool]
+    container_type: Optional[Union[List, Tuple]]
 
 
 TTableSchemaColumns = Dict[str, TColumnSchema]

dlt/extract/hints.py

@@ -279,6 +279,12 @@ def new_table_template(
         clean_columns = columns
         if columns is not None:
             clean_columns = ensure_table_schema_columns_hint(columns)
+            # TODO: extract all schema items with container_type set
+            #       and and create tables
+            for column_name, column_schema in clean_columns.items():
+                if column_schema.get("container_type") is not None:
+                    clean_columns.pop(column_name)
+
             if not callable(clean_columns):
                 clean_columns = clean_columns.values()  # type: ignore
         # create a table schema template where hints can be functions taking TDataItem