clean up compose files a bit

.github/workflows/test_doc_snippets.yml

@@ -60,7 +60,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker compose -f ".github/weaviate-compose.yml" up -d
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/test_local_destinations.yml

@@ -73,7 +73,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker compose -f ".github/weaviate-compose.yml" up -d
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -122,7 +122,7 @@ jobs:
 
       - name: Stop weaviate
         if: always()
-        run: docker compose -f ".github/weaviate-compose.yml" down -v
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" down -v
 
       - name: Stop SFTP server
         if: always()

Makefile

@@ -109,4 +109,10 @@ test-build-images: build-library
 
 preprocess-docs: 
 	# run docs preprocessing to run a few checks and ensure examples can be parsed
-	cd docs/website && npm i && npm run preprocess-docs
+	cd docs/website && npm i && npm run preprocess-docs
+
+start-test-containers:
+	docker compose -f "tests/load/dremio/docker-compose.yml" up -d
+	docker compose -f "tests/load/postgres/docker-compose.yml" up -d
+	docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
+	docker compose -f "tests/load/filesystem_sftp/docker-compose.yml" up -d

dlt/common/destination/reference.py

@@ -471,7 +471,7 @@ def create_followup_jobs(self, final_state: TLoadJobState) -> List[FollowupJobRe
 class SupportsReadableRelation(Protocol):
     """A readable relation retrieved from a destination that supports it"""
 
-    columns: TTableSchemaColumns
+    schema_columns: TTableSchemaColumns
     """Known dlt table columns for this relation"""
 
     def df(self, chunk_size: int = None) -> Optional[DataFrame]:

tests/load/utils.py

@@ -162,7 +162,7 @@ class DestinationTestConfiguration:
     supports_dbt: bool = True
     disable_compression: bool = False
     dev_mode: bool = False
-    credentials: Optional[Union[CredentialsConfiguration, Dict[str, Any]]] = None
+    credentials: Optional[Union[CredentialsConfiguration, Dict[str, Any], str]] = None
     env_vars: Optional[Dict[str, str]] = None
     destination_name: Optional[str] = None
 
@@ -215,8 +215,11 @@ def setup(self) -> None:
             os.environ["DATA_WRITER__DISABLE_COMPRESSION"] = "True"
 
         if self.credentials is not None:
-            for key, value in dict(self.credentials).items():
-                os.environ[f"DESTINATION__CREDENTIALS__{key.upper()}"] = str(value)
+            if isinstance(self.credentials, str):
+                os.environ[f"DESTINATION__CREDENTIALS"] = self.credentials
+            else:
+                for key, value in dict(self.credentials).items():
+                    os.environ[f"DESTINATION__CREDENTIALS__{key.upper()}"] = str(value)
 
         if self.env_vars is not None:
             for k, v in self.env_vars.items():
@@ -340,6 +343,7 @@ def destinations_configs(
                 supports_merge=False,
                 supports_dbt=False,
                 destination_name="sqlalchemy_sqlite",
+                credentials="sqlite:///_storage/dl_data.sqlite"
             ),
         ]