From 708c96e4de7c08b32acd0aa16cee8f2ce5f0b7b5 Mon Sep 17 00:00:00 2001 From: Joscha Feth Date: Tue, 26 Nov 2024 01:08:30 +0000 Subject: [PATCH] docs: document that `path` can also be a URL --- dlt/sources/helpers/rest_client/client.py | 14 +++++++----- .../verified-sources/rest_api/basic.md | 3 ++- .../helpers/rest_client/test_client.py | 22 +++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/dlt/sources/helpers/rest_client/client.py b/dlt/sources/helpers/rest_client/client.py index 6d04373d8d..4b589813fc 100644 --- a/dlt/sources/helpers/rest_client/client.py +++ b/dlt/sources/helpers/rest_client/client.py @@ -96,18 +96,18 @@ def __init__( def _create_request( self, - path: str, + path_or_url: str, method: HTTPMethod, params: Optional[Dict[str, Any]] = None, json: Optional[Dict[str, Any]] = None, auth: Optional[AuthBase] = None, hooks: Optional[Hooks] = None, ) -> Request: - parsed_url = urlparse(path) + parsed_url = urlparse(path_or_url) if parsed_url.scheme in ("http", "https"): - url = path + url = path_or_url else: - url = join_url(self.base_url, path) + url = join_url(self.base_url, path_or_url) return Request( method=method, @@ -140,7 +140,7 @@ def _send_request(self, request: Request, **kwargs: Any) -> Response: def request(self, path: str = "", method: HTTPMethod = "GET", **kwargs: Any) -> Response: prepared_request = self._create_request( - path=path, + path_or_url=path, method=method, params=kwargs.pop("params", None), json=kwargs.pop("json", None), @@ -171,6 +171,8 @@ def paginate( Args: path (str): Endpoint path for the request, relative to `base_url`. + Can also be a fully qualified URL; if starting with http(s) it will + be used instead of the base_url + path. method (HTTPMethodBasic): HTTP method for the request, defaults to 'get'. params (Optional[Dict[str, Any]]): URL parameters for the request. json (Optional[Dict[str, Any]]): JSON payload for the request. @@ -210,7 +212,7 @@ def paginate( hooks["response"] = [raise_for_status] request = self._create_request( - path=path, method=method, params=params, json=json, auth=auth, hooks=hooks + path_or_url=path, method=method, params=params, json=json, auth=auth, hooks=hooks ) if paginator: diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md index c600d64dbe..c76a990feb 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/rest_api/basic.md @@ -335,7 +335,8 @@ The endpoint configuration defines how to query the API endpoint. Quick example: The fields in the endpoint configuration are: -- `path`: The path to the API endpoint. +- `path`: The path to the API endpoint. By default this path is appended to the given `base_url`. If this is a fully qualified URL starting with `http:` or `https:` it will be +used as-is and `base_url` will be ignored. - `method`: The HTTP method to be used. The default is `GET`. - `params`: Query parameters to be sent with each request. For example, `sort` to order the results or `since` to specify [incremental loading](#incremental-loading). This is also used to define [resource relationships](#define-resource-relationships). - `json`: The JSON payload to be sent with the request (for POST and PUT requests). diff --git a/tests/sources/helpers/rest_client/test_client.py b/tests/sources/helpers/rest_client/test_client.py index 488d7ef525..36fe009b93 100644 --- a/tests/sources/helpers/rest_client/test_client.py +++ b/tests/sources/helpers/rest_client/test_client.py @@ -7,6 +7,7 @@ from requests import PreparedRequest, Request, Response from requests.auth import AuthBase from requests.exceptions import HTTPError +import requests_mock from dlt.common import logger from dlt.common.typing import TSecretStrValue @@ -512,3 +513,24 @@ def test_request_kwargs(self, mocker) -> None: "timeout": 432, "allow_redirects": False, } + + @requests_mock.Mocker(kw="mock") + def test_overwrite_path(self, mocker, **kwargs) -> None: + expected = {"foo": "bar"} + kwargs["mock"].get("https://completely.different/endpoint", json=expected) + rest_client = RESTClient( + base_url="https://api.example.com", + ) + response = rest_client.get("https://completely.different/endpoint") + assert response.json() == expected + + @requests_mock.Mocker(kw="mock") + def test_overwrite_path_ignores_different_protocol(self, mocker, **kwargs) -> None: + expected = {"foo": "bar"} + base_url = "https://api.example.com" + kwargs["mock"].get(f"{base_url}/my://protocol", json=expected) + rest_client = RESTClient( + base_url=base_url, + ) + response = rest_client.get("my://protocol") + assert response.json() == expected