From b1e0f7760920355050b9c1dd6324ece695830434 Mon Sep 17 00:00:00 2001 From: David Scharf Date: Tue, 21 May 2024 21:28:30 +0200 Subject: [PATCH] Fix snippet linting errors (#1392) * fix snippets * fix additional mypy errors * fix another auth type checker error --- .../verified-sources/rest_api.md | 84 +++++++++++-------- .../docs/general-usage/http/rest-client.md | 4 +- .../docs/walkthroughs/create-a-pipeline.md | 2 +- 3 files changed, 50 insertions(+), 40 deletions(-) diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md index 1f79055d06..0022850987 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api.md @@ -203,7 +203,7 @@ For example, you can set the primary key, write disposition, and other default s ```py config = { "client": { - ... + # ... }, "resource_defaults": { "primary_key": "id", @@ -216,15 +216,17 @@ config = { }, "resources": [ "resource1", - "resource2": { - "name": "resource2_name", - "write_disposition": "append", - "endpoint": { - "params": { - "param1": "value1", + { + "resource2": { + "name": "resource2_name", + "write_disposition": "append", + "endpoint": { + "params": { + "param1": "value1", + }, }, - }, - }, + } + } ], } ``` @@ -309,7 +311,7 @@ To specify the pagination configuration, use the `paginator` field in the [clien ```py { - ... + # ... "paginator": { "type": "json_links", "next_url_path": "paging.next", @@ -321,7 +323,7 @@ Or using the paginator instance: ```py { - ... + # ... "paginator": JSONResponsePaginator( next_url_path="paging.next" ), @@ -394,11 +396,11 @@ One of the most common method is token-based authentication. To authenticate wit ```py { "client": { - ... + # ... "auth": { "token": dlt.secrets["your_api_token"], }, - ... + # ... }, } ``` @@ -424,7 +426,7 @@ To specify the authentication configuration, use the `auth` field in the [client "type": "bearer", "token": dlt.secrets["your_api_token"], }, - ... + # ... }, } ``` @@ -438,7 +440,7 @@ config = { "client": { "auth": BearTokenAuth(dlt.secrets["your_api_token"]), }, - ... + # ... } ``` @@ -455,7 +457,7 @@ In the GitHub example, the `issue_comments` resource depends on the `issues` res "name": "issues", "endpoint": { "path": "issues", - ... + # ... }, }, { @@ -495,10 +497,12 @@ The `issue_comments` resource will make requests to the following endpoints: The syntax for the `resolve` field in parameter configuration is: ```py -"": { - "type": "resolve", - "resource": "", - "field": "", +{ + "": { + "type": "resolve", + "resource": "", + "field": "", + } } ``` @@ -530,21 +534,25 @@ When the API endpoint supports incremental loading, you can configure the source 1. Defining a special parameter in the `params` section of the [endpoint configuration](#endpoint-configuration): ```py - "": { - "type": "incremental", - "cursor_path": "", - "initial_value": "", - }, + { + "": { + "type": "incremental", + "cursor_path": "", + "initial_value": "", + }, + } ``` For example, in the `issues` resource configuration in the GitHub example, we have: ```py - "since": { - "type": "incremental", - "cursor_path": "updated_at", - "initial_value": "2024-01-25T11:21:28Z", - }, + { + "since": { + "type": "incremental", + "cursor_path": "updated_at", + "initial_value": "2024-01-25T11:21:28Z", + }, + } ``` This configuration tells the source to create an incremental object that will keep track of the `updated_at` field in the response and use it as a value for the `since` parameter in subsequent requests. @@ -552,13 +560,15 @@ When the API endpoint supports incremental loading, you can configure the source 2. Specifying the `incremental` field in the [endpoint configuration](#endpoint-configuration): ```py - "incremental": { - "start_param": "", - "end_param": "", - "cursor_path": "", - "initial_value": "", - "end_value": "", - }, + { + "incremental": { + "start_param": "", + "end_param": "", + "cursor_path": "", + "initial_value": "", + "end_value": "", + } + } ``` This configuration is more flexible and allows you to specify the start and end conditions for the incremental loading. diff --git a/docs/website/docs/general-usage/http/rest-client.md b/docs/website/docs/general-usage/http/rest-client.md index 3f29182044..19cc95bf78 100644 --- a/docs/website/docs/general-usage/http/rest-client.md +++ b/docs/website/docs/general-usage/http/rest-client.md @@ -542,7 +542,7 @@ from dlt.sources.helpers.rest_client import RESTClient from dlt.sources.helpers.rest_client.auth import BearerTokenAuth client = RESTClient(base_url="https://api.example.com") -response = client.get("/posts", auth=BearerTokenAuth(token="your_access_token")) +response = client.get("/posts", auth=BearerTokenAuth(token="your_access_token")) # type: ignore print(response.status_code) print(response.headers) @@ -589,7 +589,7 @@ def response_hook(response, **kwargs): for page in client.paginate( "/posts", - auth=BearerTokenAuth(token="your_access_token"), + auth=BearerTokenAuth(token="your_access_token"), # type: ignore hooks={"response": [response_hook]} ): print(page) diff --git a/docs/website/docs/walkthroughs/create-a-pipeline.md b/docs/website/docs/walkthroughs/create-a-pipeline.md index bba78dc6cb..cbbbd73fc3 100644 --- a/docs/website/docs/walkthroughs/create-a-pipeline.md +++ b/docs/website/docs/walkthroughs/create-a-pipeline.md @@ -100,7 +100,7 @@ def github_api_resource(api_secret_key: str = dlt.secrets.value): for page in paginate( url, - auth=BearerTokenAuth(api_secret_key), + auth=BearerTokenAuth(api_secret_key), # type: ignore paginator=HeaderLinkPaginator(), params={"state": "open"} ):