From 534ccb1c9293c5cd2794b85ab4bf1fd2c8ba58fd Mon Sep 17 00:00:00 2001 From: Purushottam Khedre Date: Wed, 25 Oct 2023 20:50:12 +0530 Subject: [PATCH] :sparkles: Younium: migrating Python CDK to Low-Code (#31690) Co-authored-by: Marcos Marx Co-authored-by: marcosmarxm --- .../connectors/source-younium/.dockerignore | 6 - .../connectors/source-younium/Dockerfile | 38 ----- .../connectors/source-younium/README.md | 151 ++++++++--------- .../{unit_tests => }/__init__.py | 0 .../source-younium/acceptance-test-config.yml | 17 +- .../connectors/source-younium/metadata.yaml | 25 +-- .../source-younium/requirements.txt | 2 +- .../connectors/source-younium/setup.py | 5 +- .../source_younium/components.py | 86 ++++++++++ .../source_younium/manifest.yaml | 124 ++++++++++++++ .../source_younium/schemas/invoice.json | 9 + .../source_younium/schemas/product.json | 6 + .../source_younium/schemas/subscription.json | 12 ++ .../source-younium/source_younium/source.py | 156 ++---------------- .../source-younium/source_younium/spec.yaml | 28 ---- .../source-younium/unit_tests/test_source.py | 51 ------ .../source-younium/unit_tests/test_streams.py | 73 -------- docs/integrations/sources/younium.md | 1 + 18 files changed, 347 insertions(+), 443 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-younium/.dockerignore delete mode 100644 airbyte-integrations/connectors/source-younium/Dockerfile rename airbyte-integrations/connectors/source-younium/{unit_tests => }/__init__.py (100%) create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/components.py create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-younium/source_younium/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-younium/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-younium/.dockerignore b/airbyte-integrations/connectors/source-younium/.dockerignore deleted file mode 100644 index cebe350aeec3..000000000000 --- a/airbyte-integrations/connectors/source-younium/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_younium -!setup.py -!secrets \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/Dockerfile b/airbyte-integrations/connectors/source-younium/Dockerfile deleted file mode 100644 index eef94d461602..000000000000 --- a/airbyte-integrations/connectors/source-younium/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_younium ./source_younium - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/source-younium diff --git a/airbyte-integrations/connectors/source-younium/README.md b/airbyte-integrations/connectors/source-younium/README.md index e9f0407ff831..1d4b30fdcb5f 100644 --- a/airbyte-integrations/connectors/source-younium/README.md +++ b/airbyte-integrations/connectors/source-younium/README.md @@ -1,45 +1,13 @@ # Younium Source -This is the repository for the Younium source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/younium). +This is the repository for the Younium configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/younium). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-younium:build -``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/younium) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/younium) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_younium/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +15,69 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source younium test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-younium:dev +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-younium build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-younium:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container -You can also build the connector image via Gradle: + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-younium:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-younium:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-younium:dev . +# Running the spec command against your patched connector +docker run airbyte/source-younium:dev spec #### Run Then run any of the connector commands as follows: @@ -79,44 +88,20 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-younium:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-younium:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-younium:unitTest -``` -To run acceptance and custom integration tests: +To run your integration tests with Docker, run: ``` -./gradlew :airbyte-integrations:connectors:source-younium:integrationTest +./acceptance-test-docker.sh ``` +### Using `airbyte-ci` to run tests +See [airbyte-ci documentation](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command) + + ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -129,4 +114,4 @@ You've checked out the repo, implemented a million dollar feature, and you're re 1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). 1. Create a Pull Request. 1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/__init__.py b/airbyte-integrations/connectors/source-younium/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-younium/unit_tests/__init__.py rename to airbyte-integrations/connectors/source-younium/__init__.py diff --git a/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml b/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml index ba28604f83e6..a8a1f5ebcfb4 100644 --- a/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml @@ -19,9 +19,20 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-younium/metadata.yaml b/airbyte-integrations/connectors/source-younium/metadata.yaml index 6f23d89022f3..892a1ee423c0 100644 --- a/airbyte-integrations/connectors/source-younium/metadata.yaml +++ b/airbyte-integrations/connectors/source-younium/metadata.yaml @@ -1,24 +1,27 @@ data: + registries: + oss: + enabled: true + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source definitionId: 9c74c2d7-531a-4ebf-b6d8-6181f805ecdc - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-younium githubIssueLabel: source-younium icon: younium.svg license: MIT name: Younium - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2022-11-09 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/younium tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-younium/requirements.txt b/airbyte-integrations/connectors/source-younium/requirements.txt index ecf975e2fa63..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-younium/requirements.txt +++ b/airbyte-integrations/connectors/source-younium/requirements.txt @@ -1 +1 @@ --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-younium/setup.py b/airbyte-integrations/connectors/source-younium/setup.py index b2a5f3f95a39..2a8872be5287 100644 --- a/airbyte-integrations/connectors/source-younium/setup.py +++ b/airbyte-integrations/connectors/source-younium/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", - "responses~=0.22.0", ] setup( diff --git a/airbyte-integrations/connectors/source-younium/source_younium/components.py b/airbyte-integrations/connectors/source-younium/source_younium/components.py new file mode 100644 index 000000000000..7b9bcaa42a5f --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/components.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from http import HTTPStatus +from typing import Any, Mapping, Union + +import requests +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.interpolation import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config +from requests import HTTPError + +# https://developers.zoom.us/docs/internal-apps/s2s-oauth/#successful-response +# The Bearer token generated by server-to-server token will expire in one hour + + +@dataclass +class CustomYouniumAuthenticator(NoAuth): + config: Config + + username: Union[InterpolatedString, str] + password: Union[InterpolatedString, str] + legal_entity: Union[InterpolatedString, str] + grant_type: Union[InterpolatedString, str] + client_id: Union[InterpolatedString, str] + scope: Union[InterpolatedString, str] + + _access_token = None + _token_type = None + + def __post_init__(self, parameters: Mapping[str, Any]): + self._username = InterpolatedString.create(self.username, parameters=parameters).eval(self.config) + self._password = InterpolatedString.create(self.password, parameters=parameters).eval(self.config) + self._legal_entity = InterpolatedString.create(self.legal_entity, parameters=parameters).eval(self.config) + self._grant_type = InterpolatedString.create(self.grant_type, parameters=parameters).eval(self.config) + self._client_id = InterpolatedString.create(self.client_id, parameters=parameters).eval(self.config) + self._scope = InterpolatedString.create(self.scope, parameters=parameters).eval(self.config) + + def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: + """Attach the page access token to params to authenticate on the HTTP request""" + if self._access_token is None or self._token_type is None: + self._access_token, self._token_type = self.generate_access_token() + + headers = {self.auth_header: f"{self._token_type} {self._access_token}", "Content-type": "application/json"} + + request.headers.update(headers) + + return request + + @property + def auth_header(self) -> str: + return "Authorization" + + @property + def token(self) -> str: + return self._access_token + + def generate_access_token(self) -> (str, str): + # return (str("token123"), str("Bearer")) + try: + headers = {"Content-Type": "application/x-www-form-urlencoded"} + + data = { + "username": self._username, + "password": self._password, + "legal_entity": self._legal_entity, + "grant_type": self._grant_type, + "client_id": self._client_id, + "scope": self._scope, + } + + if self.config.get("playground"): + url = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" + # url = "http://localhost:3000/playground/auth/token" + else: + url = "https://younium-identity-server.azurewebsites.net/connect/token" + # url = "http://localhost:3000/auth/token" + + rest = requests.post(url, headers=headers, data=data) + if rest.status_code != HTTPStatus.OK: + raise HTTPError(rest.text) + return (rest.json().get("access_token"), rest.json().get("token_type")) + except Exception as e: + raise Exception(f"Error while generating access token: {e}") from e diff --git a/airbyte-integrations/connectors/source-younium/source_younium/manifest.yaml b/airbyte-integrations/connectors/source-younium/source_younium/manifest.yaml new file mode 100644 index 000000000000..46d10c904aab --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/manifest.yaml @@ -0,0 +1,124 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + requester: + type: HttpRequester + url_base: "{{ 'https://apisandbox.younium.com' if config['playground'] else 'https://api.younium.com' }}" + http_method: "GET" + + authenticator: + class_name: source_younium.components.CustomYouniumAuthenticator + username: "{{ config['username'] }}" + password: "{{ config['password'] }}" + legal_entity: "{{ config['legal_entity'] }}" + grant_type: password + client_id: apiclient + scope: openid youniumapi profile + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("nextPage", {}) }}' + stop_condition: '{{ not response.get("nextPage", {}) }}' + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: PageSize + + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + account_stream: + $ref: "#/definitions/base_stream" + name: account + $parameters: + path: Accounts + + booking_stream: + $ref: "#/definitions/base_stream" + name: booking + $parameters: + path: Bookings + + invoice_stream: + $ref: "#/definitions/base_stream" + name: invoice + $parameters: + path: Invoices + + product_stream: + $ref: "#/definitions/base_stream" + name: product + $parameters: + path: Products + + subscription_stream: + $ref: "#/definitions/base_stream" + name: subscription + $parameters: + path: Subscriptions + +streams: + - "#/definitions/account_stream" + - "#/definitions/booking_stream" + - "#/definitions/invoice_stream" + - "#/definitions/product_stream" + - "#/definitions/subscription_stream" + +check: + type: CheckStream + stream_names: + - account + - booking + - invoice + - product + - subscription +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/younium + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Younium Spec + type: object + additionalProperties: true + required: + - username + - password + - legal_entity + properties: + username: + title: Username + type: string + description: Username for Younium account + password: + title: Password + type: string + description: Account password for younium account API key + airbyte_secret: true + legal_entity: + title: Legal Entity + type: string + description: Legal Entity that data should be pulled from + playground: + title: Playground environment + type: boolean + description: Property defining if connector is used against playground or production environment + default: false diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json index 6c78eec6145e..26de8b306f75 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json @@ -12,6 +12,15 @@ "status": { "type": ["null", "string"] }, + "created": { + "type": ["null", "string"] + }, + "invoiceDeliveryMethod": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] + }, "account": { "type": "object", "properties": { diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json index 3e8f8315966c..d3f55669708d 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json @@ -12,6 +12,12 @@ "name": { "type": ["null", "string"] }, + "created": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] + }, "productType": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json index 4022cdd1dda0..60a664830f81 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json @@ -9,6 +9,18 @@ "orderNumber": { "type": ["null", "string"] }, + "created": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] + }, + "orderBillingPeriod": { + "type": ["null", "string"] + }, + "setOrderBillingPeriod": { + "type": ["null", "boolean"] + }, "version": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-younium/source_younium/source.py b/airbyte-integrations/connectors/source-younium/source_younium/source.py index f11c5b2eef51..2b8e59669dbf 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/source.py +++ b/airbyte-integrations/connectors/source-younium/source_younium/source.py @@ -2,153 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class YouniumStream(HttpStream, ABC): - # url_base = "https://apisandbox.younium.com" - - # https://api.younium.com - def __init__(self, authenticator=TokenAuthenticator, playground: bool = False, *args, **kwargs): - super().__init__(authenticator=authenticator) - self.page_size = 100 - self.playground: bool = playground - - @property - def url_base(self) -> str: - if self.playground: - endpoint = "https://apisandbox.younium.com" - else: - endpoint = "https://api.younium.com" - return endpoint - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response = response.json() - current_page = response.get("pageNumber", 1) - total_rows = response.get("totalCount", 0) - - total_pages = total_rows // self.page_size - - if current_page <= total_pages: - return {"pageNumber": current_page + 1} - else: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - if next_page_token: - return {"pageNumber": next_page_token["pageNumber"], "PageSize": self.page_size} - else: - return {"PageSize": self.page_size} - - def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - response_results = response.json() - yield from response_results.get("data", []) - - -class Account(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Accounts" - - -class Booking(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Bookings" - - -class Invoice(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Invoices" - - -class Product(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Products" - - -class Subscription(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Subscriptions" - - -class SourceYounium(AbstractSource): - def get_auth(self, config): - scope = "openid youniumapi profile" - - if config.get("playground"): - url = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" - else: - url = "https://younium-identity-server.azurewebsites.net/connect/token" - - payload = f"grant_type=password&client_id=apiclient&username={config['username']}&password={config['password']}&scope={scope}" - headers = {"Content-Type": "application/x-www-form-urlencoded"} - response = requests.request("POST", url, headers=headers, data=payload) - response.raise_for_status() - access_token = response.json()["access_token"] - - auth = TokenAuthenticator(token=access_token) - return auth - - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - stream = Invoice(authenticator=self.get_auth(config), **config) - stream.next_page_token = lambda response: None - stream.page_size = 1 - # auth = self.get_auth(config) - _ = list(stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - logger.error(e) - return False, repr(e) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = self.get_auth(config) - return [ - Account(authenticator=auth, **config), - Booking(authenticator=auth, **config), - Invoice(authenticator=auth, **config), - Product(authenticator=auth, **config), - Subscription(authenticator=auth, **config), - ] +# Declarative Source +class SourceYounium(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml b/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml deleted file mode 100644 index 0cf38ff69a03..000000000000 --- a/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml +++ /dev/null @@ -1,28 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/younium -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Younium Spec - type: object - required: - - username - - password - - legal_entity - properties: - username: - title: Username - type: string - description: Username for Younium account - password: - title: Password - type: string - description: Account password for younium account API key - airbyte_secret: true - legal_entity: - title: Legal Entity - type: string - description: Legal Entity that data should be pulled from - playground: - title: Playground environment - type: boolean - description: Property defining if connector is used against playground or production environment - default: false diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py b/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py deleted file mode 100644 index 50e9e6806847..000000000000 --- a/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import responses -from source_younium.source import SourceYounium - - -@responses.activate -def test_check_connection(mocker): - sandbox = False - - source = SourceYounium() - # mock the post request - - if sandbox: - mock_url1 = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" - mock_url2 = "https://apisandbox.younium.com/Invoices?PageSize=1" - else: - mock_url1 = "https://younium-identity-server.azurewebsites.net/connect/token" - mock_url2 = "https://api.younium.com/Invoices?PageSize=1" - # Mock the POST to get the access token - responses.add( - responses.POST, - mock_url1, - json={ - "access_token": "dummy_token", - }, - status=HTTPStatus.OK, - ) - - # Mock the GET to get the first page of the stream - responses.add(responses.GET, mock_url2, json={}, status=HTTPStatus.OK) - - logger_mock = MagicMock() - config_mock = {"playground": sandbox, "username": "dummy_username", "password": "dummy_password"} - - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceYounium() - mocker.patch.object(source, "get_auth", return_value="dummy_token") - config_mock = {"playground": False, "username": "dummy_username", "password": "dummy_password"} - streams = source.streams(config_mock) - - expected_streams_number = 5 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py deleted file mode 100644 index ab0226d7c9b2..000000000000 --- a/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_younium.source import YouniumStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(YouniumStream, "path", "v0/example_endpoint") - mocker.patch.object(YouniumStream, "primary_key", "test_primary_key") - mocker.patch.object(YouniumStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = YouniumStream(authenticator=None) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"PageSize": 100} - assert stream.request_params(**inputs) == expected_params - - -def test_request_params_with_next_page_token(patch_base_class): - stream = YouniumStream(authenticator=None) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"pageNumber": 2}} - expected_params = {"PageSize": 100, "pageNumber": 2} - assert stream.request_params(**inputs) == expected_params - - -def test_playground_url_base(patch_base_class): - stream = YouniumStream(authenticator=None, playground=True) - expected_url_base = "https://apisandbox.younium.com" - assert stream.url_base == expected_url_base - - -def test_use_playground_url_base(patch_base_class): - stream = YouniumStream(authenticator=None, playground=True) - expected_url_base = "https://apisandbox.younium.com" - assert stream.url_base == expected_url_base - - -def test_http_method(patch_base_class): - stream = YouniumStream(authenticator=None) - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = YouniumStream(authenticator=None) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = YouniumStream(authenticator=None) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/younium.md b/docs/integrations/sources/younium.md index 85f38eb4b462..b3e242b80fa4 100644 --- a/docs/integrations/sources/younium.md +++ b/docs/integrations/sources/younium.md @@ -43,5 +43,6 @@ The Younium source connector supports the following [sync modes](https://docs.ai | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- |:---------------------------------------------------| +| 0.3.0 | 2023-10-25 | [31690](https://github.com/airbytehq/airbyte/pull/31690) | Migrate to low-code framework | | 0.2.0 | 2023-03-29 | [24655](https://github.com/airbytehq/airbyte/pull/24655) | Source Younium: Adding Booking and Account streams | | 0.1.0 | 2022-11-09 | [18758](https://github.com/airbytehq/airbyte/pull/18758) | 🎉 New Source: Younium [python cdk] |