From 90d54144cbb741d238c4c833dfede47c62c94194 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 10:42:12 +0200 Subject: [PATCH 01/16] add SelectiveAuthenticator --- .../auth/selective_authenticator.py | 29 ++++++++++++++ .../declarative_component_schema.yaml | 33 ++++++++++++++++ .../auth/test_selective_authenticator.py | 39 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py create mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py new file mode 100644 index 000000000000..803af7103382 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, Mapping, List + +import dpath + +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator + + +@dataclass +class SelectiveAuthenticator(DeclarativeAuthenticator): + """Authenticator that selects concrete implementation based on specific config value""" + config: Mapping[str, Any] + authenticators: List[DeclarativeAuthenticator] + authenticator_selection_path: List[str] + + def __new__(cls, config, authenticators, authenticator_selection_path, *args, **kwargs): + try: + selected_key = str(dpath.util.get(config, authenticator_selection_path)) + except KeyError as err: + raise ValueError("The path from `authenticator_selection_path` is not found in the config.") from err + + try: + return authenticators[selected_key] + except KeyError as err: + raise ValueError(f"The authenticator `{selected_key}` is not found.") from err diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 8d4af292f68a..aa11c6a8eddc 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -224,6 +224,38 @@ definitions: $parameters: type: object additionalProperties: true + SelectiveAuthenticator: + title: Selective Authenticator + description: Authenticator component whose behavior is derived from a custom code implementation of the connector. + type: object + additionalProperties: true + required: + - type + - authenticators + - authenticator_selection_path + properties: + type: + type: string + enum: [ SelectiveAuthenticator ] + authenticators: + title: Authenticators + description: Authenticators to select from. + type: object + additionalProperties: + anyOf: + - "$ref": "#/definitions/ApiKeyAuthenticator" + - "$ref": "#/definitions/BasicHttpAuthenticator" + - "$ref": "#/definitions/BearerAuthenticator" + - "$ref": "#/definitions/CustomAuthenticator" + - "$ref": "#/definitions/OAuthAuthenticator" + - "$ref": "#/definitions/NoAuth" + - "$ref": "#/definitions/SessionTokenAuthenticator" + - "$ref": "#/definitions/LegacySessionTokenAuthenticator" + examples: + - "TODO" + $parameters: + type: object + additionalProperties: true CheckStream: title: Streams to Check description: Defines the streams to try reading when running a check operation. @@ -1149,6 +1181,7 @@ definitions: - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" + - "$ref": "#/definitions/SelectiveAuthenticator" error_handler: title: Error Handler description: Error handler component that defines how to handle errors. diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py new file mode 100644 index 000000000000..0c105a322efd --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator +import pytest + + +def test_authenticator_selected(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + auth = SelectiveAuthenticator( + config={"auth": {"type": "one"}}, + authenticators=authenticators, + authenticator_selection_path=["auth", "type"], + ) + + assert auth is authenticators["one"] + + +def test_selection_path_not_found(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + + with pytest.raises(ValueError, match="The path from `authenticator_selection_path` is not found in the config"): + _ = SelectiveAuthenticator( + config={"auth": {"type": "one"}}, + authenticators=authenticators, + authenticator_selection_path=["auth_type"], + ) + + +def test_selected_auth_not_found(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + + with pytest.raises(ValueError, match="The authenticator `unknown` is not found"): + _ = SelectiveAuthenticator( + config={"auth": {"type": "unknown"}}, + authenticators=authenticators, + authenticator_selection_path=["auth", "type"], + ) From a4e9fc935cace6576b8697091b909d5af77736e3 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 13:46:35 +0200 Subject: [PATCH 02/16] format --- .../sources/declarative/auth/selective_authenticator.py | 6 +++--- .../sources/declarative/declarative_component_schema.yaml | 4 ++-- .../declarative/auth/test_selective_authenticator.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py index 803af7103382..28e35074f5c9 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -3,16 +3,16 @@ # from dataclasses import dataclass -from typing import Any, Mapping, List +from typing import Any, List, Mapping import dpath - from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator @dataclass class SelectiveAuthenticator(DeclarativeAuthenticator): - """Authenticator that selects concrete implementation based on specific config value""" + """Authenticator that selects concrete implementation based on specific config value.""" + config: Mapping[str, Any] authenticators: List[DeclarativeAuthenticator] authenticator_selection_path: List[str] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index aa11c6a8eddc..5e37fd62e6b2 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -226,7 +226,7 @@ definitions: additionalProperties: true SelectiveAuthenticator: title: Selective Authenticator - description: Authenticator component whose behavior is derived from a custom code implementation of the connector. + description: Authenticator that selects concrete authenticator based on config property. type: object additionalProperties: true required: @@ -236,7 +236,7 @@ definitions: properties: type: type: string - enum: [ SelectiveAuthenticator ] + enum: [SelectiveAuthenticator] authenticators: title: Authenticators description: Authenticators to select from. diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py index 0c105a322efd..346b284c3786 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py @@ -2,8 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator import pytest +from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator def test_authenticator_selected(mocker): From 93a16dc9bf4c34de03a2d4c8caa519403799e351 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 13:50:46 +0200 Subject: [PATCH 03/16] mypy --- .../declarative/auth/selective_authenticator.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py index 28e35074f5c9..6b105affcdf7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -14,10 +14,17 @@ class SelectiveAuthenticator(DeclarativeAuthenticator): """Authenticator that selects concrete implementation based on specific config value.""" config: Mapping[str, Any] - authenticators: List[DeclarativeAuthenticator] + authenticators: List[str, DeclarativeAuthenticator] authenticator_selection_path: List[str] - def __new__(cls, config, authenticators, authenticator_selection_path, *args, **kwargs): + def __new__( + cls, + config: Mapping[str, Any], + authenticators: Mapping[str, DeclarativeAuthenticator], + authenticator_selection_path: List[str], + *args, + **kwargs, + ): try: selected_key = str(dpath.util.get(config, authenticator_selection_path)) except KeyError as err: From 6ab3fe795f65975a82c3e96f56cfb76a848be7b4 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 13:52:49 +0200 Subject: [PATCH 04/16] mypy --- .../sources/declarative/auth/selective_authenticator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py index 6b105affcdf7..b706f77ee7b5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -14,7 +14,7 @@ class SelectiveAuthenticator(DeclarativeAuthenticator): """Authenticator that selects concrete implementation based on specific config value.""" config: Mapping[str, Any] - authenticators: List[str, DeclarativeAuthenticator] + authenticators: Mapping[str, DeclarativeAuthenticator] authenticator_selection_path: List[str] def __new__( From c4bbaa686194ca4dc49c44ed58acce02a73fc5d3 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 14:48:15 +0200 Subject: [PATCH 05/16] mypy --- .../sources/declarative/auth/selective_authenticator.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py index b706f77ee7b5..2e6fff582e58 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -17,14 +17,15 @@ class SelectiveAuthenticator(DeclarativeAuthenticator): authenticators: Mapping[str, DeclarativeAuthenticator] authenticator_selection_path: List[str] - def __new__( + # returns "DeclarativeAuthenticator", but must return a subtype of "SelectiveAuthenticator" + def __new__( # type: ignore[misc] cls, config: Mapping[str, Any], authenticators: Mapping[str, DeclarativeAuthenticator], authenticator_selection_path: List[str], - *args, - **kwargs, - ): + *arg: Any, + **kwargs: Any, + ) -> DeclarativeAuthenticator: try: selected_key = str(dpath.util.get(config, authenticator_selection_path)) except KeyError as err: From 122708cab76f03401228e8075b3f94ef53422b86 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 14:59:27 +0200 Subject: [PATCH 06/16] format --- .../sources/declarative/auth/selective_authenticator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py index 2e6fff582e58..6a9d6128706b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -18,7 +18,7 @@ class SelectiveAuthenticator(DeclarativeAuthenticator): authenticator_selection_path: List[str] # returns "DeclarativeAuthenticator", but must return a subtype of "SelectiveAuthenticator" - def __new__( # type: ignore[misc] + def __new__( # type: ignore[misc] cls, config: Mapping[str, Any], authenticators: Mapping[str, DeclarativeAuthenticator], From 9fdd148d691032679772b35bfc1364d20a06fd94 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 18:21:10 +0200 Subject: [PATCH 07/16] add factory --- .../parsers/model_to_component_factory.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 33138409afd2..699885b3f8fd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -11,8 +11,9 @@ from airbyte_cdk.models import Level from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator -from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator +from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator from airbyte_cdk.sources.declarative.auth.token import ( ApiKeyAuthenticator, BasicHttpAuthenticator, @@ -76,6 +77,7 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import RemoveFields as RemoveFieldsModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestOption as RequestOptionModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestPath as RequestPathModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator as SelectiveAuthenticatorModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SessionTokenAuthenticator as SessionTokenAuthenticatorModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel @@ -187,6 +189,7 @@ def _init_mappings(self) -> None: RequestPathModel: self.create_request_path, RequestOptionModel: self.create_request_option, LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, + SelectiveAuthenticatorModel: self.create_selective_authenticator, SimpleRetrieverModel: self.create_simple_retriever, SpecModel: self.create_spec, SubstreamPartitionRouterModel: self.create_substream_partition_router, @@ -896,6 +899,15 @@ def create_record_selector( def create_remove_fields(model: RemoveFieldsModel, config: Config, **kwargs: Any) -> RemoveFields: return RemoveFields(field_pointers=model.field_pointers, parameters={}) + def create_selective_authenticator(self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any) -> DeclarativeAuthenticator: + authenticators = {name: self._create_component_from_model(model=auth, config=config) for name, auth in model.authenticators.items()} + return SelectiveAuthenticator( + config=config, + authenticators=authenticators, + authenticator_selection_path=model.authenticator_selection_path, + **kwargs, + ) + @staticmethod def create_legacy_session_token_authenticator( model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any From 3b6f2af8e3255fc8ad786ceb803863875dd97c70 Mon Sep 17 00:00:00 2001 From: Yevhenii Kurochkin Date: Mon, 18 Dec 2023 09:13:53 +0000 Subject: [PATCH 08/16] run ./gradlew generateComponentManifestClassFiles --- .../models/declarative_component_schema.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 9981f26169e8..5cb2a3d67616 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1069,6 +1069,32 @@ class Config: ) +class SelectiveAuthenticator(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['SelectiveAuthenticator'] + authenticators: Dict[ + str, + Union[ + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + CustomAuthenticator, + OAuthAuthenticator, + NoAuth, + SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, + ], + ] = Field( + ..., + description='Authenticators to select from.', + examples=['TODO'], + title='Authenticators', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + class DeclarativeStream(BaseModel): class Config: extra = Extra.allow @@ -1179,6 +1205,7 @@ class HttpRequester(BaseModel): NoAuth, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, + SelectiveAuthenticator, ] ] = Field( None, @@ -1313,6 +1340,7 @@ class SubstreamPartitionRouter(BaseModel): CompositeErrorHandler.update_forward_refs() DeclarativeSource.update_forward_refs() +SelectiveAuthenticator.update_forward_refs() DeclarativeStream.update_forward_refs() SessionTokenAuthenticator.update_forward_refs() SimpleRetriever.update_forward_refs() From 2932b2e066990df36c7b78fa087929f8fc622da6 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 18 Dec 2023 11:47:16 +0200 Subject: [PATCH 09/16] fix missing authenticator_selection_path field --- .../declarative/declarative_component_schema.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 5e37fd62e6b2..e5823d07f874 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -237,6 +237,15 @@ definitions: type: type: string enum: [SelectiveAuthenticator] + authenticator_selection_path: + title: Authenticator Selection Path + description: Path of the field in config with selected authenticator name + type: array + items: + type: string + examples: + - [ "auth" ] + - [ "auth", "type" ] authenticators: title: Authenticators description: Authenticators to select from. @@ -252,7 +261,9 @@ definitions: - "$ref": "#/definitions/SessionTokenAuthenticator" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" examples: - - "TODO" + - authenticators: + token: "#/definitions/ApiKeyAuthenticator" + oauth: "#/definitions/OAuthAuthenticator" $parameters: type: object additionalProperties: true From bd3a213fc4864a9eced6d882b69c737b28f65d86 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 18 Dec 2023 12:14:50 +0200 Subject: [PATCH 10/16] run ./gradlew generateComponentManifestClassFiles --- .../models/declarative_component_schema.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 5cb2a3d67616..d83db5069d60 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1074,6 +1074,12 @@ class Config: extra = Extra.allow type: Literal['SelectiveAuthenticator'] + authenticator_selection_path: List[str] = Field( + ..., + description='Path of the field in config with selected authenticator name', + examples=[['auth'], ['auth', 'type']], + title='Authenticator Selection Path', + ) authenticators: Dict[ str, Union[ @@ -1089,7 +1095,14 @@ class Config: ] = Field( ..., description='Authenticators to select from.', - examples=['TODO'], + examples=[ + { + 'authenticators': { + 'token': '#/definitions/ApiKeyAuthenticator', + 'oauth': '#/definitions/OAuthAuthenticator', + } + } + ], title='Authenticators', ) parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') From 42312f9a0dd459b9bc9c66f7766526e0c889cc79 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 18 Dec 2023 16:11:06 +0200 Subject: [PATCH 11/16] mypy --- .../declarative/parsers/model_to_component_factory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 699885b3f8fd..df8d79f93499 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -714,6 +714,8 @@ def create_http_requester(self, model: HttpRequesterModel, config: Config, *, na model.http_method if isinstance(model.http_method, str) else model.http_method.value if model.http_method is not None else "GET" ) + assert model.use_cache is not None # for mypy + return HttpRequester( name=name, url_base=model.url_base, @@ -901,7 +903,8 @@ def create_remove_fields(model: RemoveFieldsModel, config: Config, **kwargs: Any def create_selective_authenticator(self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any) -> DeclarativeAuthenticator: authenticators = {name: self._create_component_from_model(model=auth, config=config) for name, auth in model.authenticators.items()} - return SelectiveAuthenticator( + # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error + return SelectiveAuthenticator( # type: ignore[abstract] config=config, authenticators=authenticators, authenticator_selection_path=model.authenticator_selection_path, From 1376b091f5a2182b0185096991ba648bf9daeac5 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 18 Dec 2023 16:14:41 +0200 Subject: [PATCH 12/16] format --- .../sources/declarative/declarative_component_schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index e5823d07f874..0494af7ce761 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -244,8 +244,8 @@ definitions: items: type: string examples: - - [ "auth" ] - - [ "auth", "type" ] + - ["auth"] + - ["auth", "type"] authenticators: title: Authenticators description: Authenticators to select from. From 6f053188d4ed90af5176cfc4afab50e1aae14f70 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 18 Dec 2023 19:47:14 +0200 Subject: [PATCH 13/16] add test for model_to_component.py --- .../test_model_to_component_factory.py | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index efac06b3d08b..183d50cfba21 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -37,7 +37,8 @@ from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models import Spec as SpecModel from airbyte_cdk.sources.declarative.models import SubstreamPartitionRouter as SubstreamPartitionRouterModel -from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel, \ + SelectiveAuthenticator from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver @@ -936,6 +937,52 @@ def test_create_request_with_session_authenticator(): } + +@pytest.mark.parametrize("input_config, expected_authenticator_class", + [ + pytest.param( + {"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}}, + ApiKeyAuthenticator, + id="test_create_requester_with_selective_authenticator_and_token_selected", + ), + pytest.param( + {"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}}, + DeclarativeOauth2Authenticator, + id="test_create_requester_with_selective_authenticator_and_oauth_selected", + ), + ] +) +def test_create_requester_with_selective_authenticator(input_config, expected_authenticator_class): + content = """ +authenticator: + type: SelectiveAuthenticator + authenticator_selection_path: + - auth + - type + authenticators: + token: + type: ApiKeyAuthenticator + header: "Authorization" + api_token: "api_key={{ config['credentials']['api_key'] }}" + oauth: + type: OAuthAuthenticator + token_refresh_endpoint: https://api.url.com + client_id: "{{ config['credentials']['client_id'] }}" + client_secret: some_secret + refresh_token: some_token + """ + name = "name" + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {}) + + authenticator = factory.create_component( + model_type=SelectiveAuthenticator, component_definition=authenticator_manifest, config=input_config, name=name + ) + + assert isinstance(authenticator, expected_authenticator_class) + + def test_create_composite_error_handler(): content = """ error_handler: From 367cf316fccf3f50a3e35675885ff7e9e4f06faa Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 18 Dec 2023 21:31:48 +0200 Subject: [PATCH 14/16] format --- .../declarative/parsers/test_model_to_component_factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 183d50cfba21..02370e779797 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -37,9 +37,9 @@ from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models import Spec as SpecModel from airbyte_cdk.sources.declarative.models import SubstreamPartitionRouter as SubstreamPartitionRouterModel -from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel, \ - SelectiveAuthenticator +from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory From 73192f7c204393217cb995f3ff3f42d98f3eadd7 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 18 Dec 2023 21:45:26 +0200 Subject: [PATCH 15/16] format --- .../test_model_to_component_factory.py | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 02370e779797..8aeba55ea18c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -937,20 +937,18 @@ def test_create_request_with_session_authenticator(): } - -@pytest.mark.parametrize("input_config, expected_authenticator_class", - [ - pytest.param( - {"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}}, - ApiKeyAuthenticator, - id="test_create_requester_with_selective_authenticator_and_token_selected", - ), - pytest.param( - {"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}}, - DeclarativeOauth2Authenticator, - id="test_create_requester_with_selective_authenticator_and_oauth_selected", - ), - ] +@pytest.mark.parametrize("input_config, expected_authenticator_class", [ + pytest.param( + {"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}}, + ApiKeyAuthenticator, + id="test_create_requester_with_selective_authenticator_and_token_selected", + ), + pytest.param( + {"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}}, + DeclarativeOauth2Authenticator, + id="test_create_requester_with_selective_authenticator_and_oauth_selected", + ), +] ) def test_create_requester_with_selective_authenticator(input_config, expected_authenticator_class): content = """ From edf90dbc7e82d870e291d1e2f5d996626248f4a9 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 15 Dec 2023 18:51:24 +0200 Subject: [PATCH 16/16] replace custom authenticator with SelectiveAuthenticator --- .../source_retently/components.py | 23 ------------------- .../source_retently/manifest.yaml | 10 +++++--- 2 files changed, 7 insertions(+), 26 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-retently/source_retently/components.py diff --git a/airbyte-integrations/connectors/source-retently/source_retently/components.py b/airbyte-integrations/connectors/source-retently/source_retently/components.py deleted file mode 100644 index 064ca721a62d..000000000000 --- a/airbyte-integrations/connectors/source-retently/source_retently/components.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dataclasses import dataclass -from typing import Any, Mapping - -from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator -from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator -from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator - - -@dataclass -class AuthenticatorRetently(DeclarativeAuthenticator): - config: Mapping[str, Any] - api_auth: ApiKeyAuthenticator - oauth: DeclarativeOauth2Authenticator - - def __new__(cls, api_auth, oauth, config, *args, **kwargs): - if config["credentials"]["api_key"]: - return api_auth - else: - return oauth diff --git a/airbyte-integrations/connectors/source-retently/source_retently/manifest.yaml b/airbyte-integrations/connectors/source-retently/source_retently/manifest.yaml index 8a8a910691da..642f1c8e2ad2 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/manifest.yaml +++ b/airbyte-integrations/connectors/source-retently/source_retently/manifest.yaml @@ -28,9 +28,13 @@ definitions: url_base: "https://app.retently.com/api/v2/" http_method: "GET" authenticator: - class_name: source_retently.components.AuthenticatorRetently - api_auth: "#/definitions/api_authenticator" - oauth: "#/definitions/oauth_authenticator" + type: SelectiveAuthenticator + authenticator_selection_path: + - auth + - type + authenticators: + token: "#/definitions/api_authenticator" + oauth: "#/definitions/oauth_authenticator" retriever: type: SimpleRetriever