From a7e04d4c110035415b4176f42ccdda84935eadad Mon Sep 17 00:00:00 2001 From: Shivam Date: Mon, 20 Nov 2023 21:12:18 -0800 Subject: [PATCH 1/4] Migrate Todoist to LOW-CODE --- .../connectors/source-todoist/.dockerignore | 6 - .../connectors/source-todoist/Dockerfile | 38 ------ .../connectors/source-todoist/README.md | 109 ++++++++++-------- .../connectors/source-todoist/__init__.py | 3 + .../source-todoist/acceptance-test-config.yml | 30 ++--- .../connectors/source-todoist/icon.svg | 6 - .../integration_tests/__init__.py | 2 +- .../integration_tests/abnormal_state.json | 36 +----- .../integration_tests/acceptance.py | 2 + .../integration_tests/configured_catalog.json | 26 ++--- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../connectors/source-todoist/metadata.yaml | 28 +++-- .../source-todoist/requirements.txt | 2 +- .../connectors/source-todoist/setup.py | 8 +- .../source-todoist/source_todoist/__init__.py | 2 +- .../source_todoist/manifest.yaml | 62 ++++++++++ .../schemas/{tasks.json => employees.json} | 2 +- .../source_todoist/schemas/projects.json | 2 +- .../source-todoist/source_todoist/source.py | 69 ++--------- .../source-todoist/source_todoist/spec.yaml | 15 --- .../source-todoist/unit_tests/__init__.py | 3 - .../source-todoist/unit_tests/test_source.py | 25 ---- .../source-todoist/unit_tests/test_streams.py | 65 ----------- 24 files changed, 191 insertions(+), 354 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-todoist/.dockerignore delete mode 100644 airbyte-integrations/connectors/source-todoist/Dockerfile create mode 100644 airbyte-integrations/connectors/source-todoist/__init__.py delete mode 100644 airbyte-integrations/connectors/source-todoist/icon.svg create mode 100644 airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml rename airbyte-integrations/connectors/source-todoist/source_todoist/schemas/{tasks.json => employees.json} (99%) delete mode 100644 airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-todoist/.dockerignore b/airbyte-integrations/connectors/source-todoist/.dockerignore deleted file mode 100644 index d853b63208a0..000000000000 --- a/airbyte-integrations/connectors/source-todoist/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_todoist -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-todoist/Dockerfile b/airbyte-integrations/connectors/source-todoist/Dockerfile deleted file mode 100644 index c30309061741..000000000000 --- a/airbyte-integrations/connectors/source-todoist/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_todoist ./source_todoist - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-todoist diff --git a/airbyte-integrations/connectors/source-todoist/README.md b/airbyte-integrations/connectors/source-todoist/README.md index bd25a8590776..b68dcdc0a9bf 100644 --- a/airbyte-integrations/connectors/source-todoist/README.md +++ b/airbyte-integrations/connectors/source-todoist/README.md @@ -1,37 +1,13 @@ # Todoist Source -This is the repository for the Todoist source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/todoist). +This is the repository for the Todoist configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/todoist). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/todoist) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/todoist) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_todoist/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -39,29 +15,69 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source todoist test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name=source-todoist build +airbyte-ci connectors --name source-todoist build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-todoist:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -An image will be built with the tag `airbyte/source-todoist:dev`. +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. -**Via `docker build`:** +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-todoist:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: ```bash docker build -t airbyte/source-todoist:dev . -``` +# Running the spec command against your patched connector +docker run airbyte/source-todoist:dev spec #### Run Then run any of the connector commands as follows: @@ -71,16 +87,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-todoist:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-todoist:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-todoist:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` - ## Testing -You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -```bash -airbyte-ci connectors --name=source-todoist test -``` -### Customizing acceptance Tests +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-todoist test +``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connectors/source-todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml index b14d04a96904..77f5556cfc2d 100644 --- a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml @@ -1,7 +1,6 @@ -# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-todoist:dev -test_strictness_level: low acceptance_tests: spec: tests: @@ -20,20 +19,21 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.txt" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" - # TODO uncomment this block this block if your connector implements incremental sync: - # tests: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/icon.svg b/airbyte-integrations/connectors/source-todoist/icon.svg deleted file mode 100644 index dbf417c99c8d..000000000000 --- a/airbyte-integrations/connectors/source-todoist/icon.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py index 1100c1c58cf5..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json index 09f16c3ccf2a..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json @@ -1,37 +1,5 @@ { - "tasks": { - "id": false, - "project_id": 10, - "section_id": -10, - "content": 10, - "description": true, - "is_completed": "not so true", - "labels": [true, false], - "parent_id": -50, - "order": true, - "priority": "10", - "due": true, - "url": 50, - "comment_count": "10", - "created_at": { "when": true }, - "creator_id": -1, - "assignee_id": 20, - "assigner_id": 50 - }, - "projects": { - "id": { - "number": "2203306141" - }, - "name": 50, - "comment_count": false, - "order": "1", - "color": 100, - "is_shared": [false], - "is_favorite": "world", - "parent_id": 100, - "is_inbox_project": [true], - "is_team_inbox": "false", - "view_style": ["list"], - "url": ["https://todoist.com/showProject?id=2203306141"] + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json index 275f28ac14f5..52caaced50a4 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json @@ -1,34 +1,22 @@ { "streams": [ { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "tasks", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "projects", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json index 37cbe64ae615..f3732995784f 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "token": "INVALID TOKEN" + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json index 55e640b7e5c3..ecc4913b84c7 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { - "token": "VALID TOKEN" + "fix-me": "TODO" } diff --git a/airbyte-integrations/connectors/source-todoist/metadata.yaml b/airbyte-integrations/connectors/source-todoist/metadata.yaml index f32560daec66..d95d9a5696ff 100644 --- a/airbyte-integrations/connectors/source-todoist/metadata.yaml +++ b/airbyte-integrations/connectors/source-todoist/metadata.yaml @@ -1,24 +1,30 @@ data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: false + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source - definitionId: 7d272065-c316-4c04-a433-cd4ee143f83e + definitionId: 1a3d38e4-dc6b-4154-b56b-582f9e978ecd dockerImageTag: 0.1.0 dockerRepository: airbyte/source-todoist githubIssueLabel: source-todoist icon: todoist.svg license: MIT name: Todoist - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/todoist tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-todoist/requirements.txt b/airbyte-integrations/connectors/source-todoist/requirements.txt index ecf975e2fa63..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-todoist/requirements.txt +++ b/airbyte-integrations/connectors/source-todoist/requirements.txt @@ -1 +1 @@ --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-todoist/setup.py b/airbyte-integrations/connectors/source-todoist/setup.py index cf672aeb4515..b01fda514b5f 100644 --- a/airbyte-integrations/connectors/source-todoist/setup.py +++ b/airbyte-integrations/connectors/source-todoist/setup.py @@ -6,14 +6,14 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6", - "requests_mock~=1.8", + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py index f04f17ce4a6f..62d0c357fa93 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml new file mode 100644 index 000000000000..92eff2ccb219 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml @@ -0,0 +1,62 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.todoist.com/rest/v2" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + tasks_stream: + $ref: "#/definitions/base_stream" + name: "tasks" + $parameters: + path: "/tasks" + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + $parameters: + path: "/projects" + + +streams: + - "#/definitions/tasks_stream" + - "#/definitions/projects_stream" + +check: + type: CheckStream + stream_names: + - "tasks" + - "projects" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/source-todolist + connection_specification: + title: Source Todolist Spec + type: object + required: + - token + additionalProperties: true + properties: + token: + type: string + description: API authorization bearer token for authenticating the API \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json similarity index 99% rename from airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json rename to airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json index cf22f8da8a4b..16d2acae12c5 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json @@ -79,4 +79,4 @@ "type": ["null", "string"] } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json index f43f7f3da3f5..b10103388dc9 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json @@ -40,4 +40,4 @@ "type": ["null", "string"] } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py index 517c10ae7383..b363f5fb2623 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py @@ -2,66 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class TodoistStream(HttpStream): - """ - Stream for Todoist REST API : https://developer.todoist.com/rest/v2/#overview - """ - @property - def url_base(self) -> str: - return "https://api.todoist.com/rest/v2/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.name.title().lower() - - -class Tasks(TodoistStream): - - primary_key = "id" - - -class Projects(TodoistStream): - - primary_key = "id" - - -# Source -class SourceTodoist(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config["token"] - authenticator = TokenAuthenticator(token=token) - task_stream = Tasks(authenticator) - task_records = task_stream.read_records(sync_mode="full_refresh") - next(task_records) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - token = config["token"] - auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support - return [Tasks(authenticator=auth), Projects(authenticator=auth)] +# Declarative Source +class SourceTodoist(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml deleted file mode 100644 index 19e7f9dc2e42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -documentationUrl: https://docs.airbyte.io/integrations/sources/todoist -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Todoist Spec - type: object - required: - - token - properties: - token: - type: string - description: >- - Your API Token. See here. The token is - case sensitive. - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py deleted file mode 100644 index d4dbafa573be..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_todoist.source import SourceTodoist - - -def test_check_connection(mocker): - source = SourceTodoist() - fake_info_record = {"collection": "is_mocked"} - with patch("source_todoist.source.Tasks.read_records", MagicMock(return_value=iter([fake_info_record]))): - logger_mock = MagicMock() - config_mock = {"token": "test"} - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceTodoist() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py deleted file mode 100644 index af1ed528da42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_todoist.source import TodoistStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(TodoistStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = TodoistStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = TodoistStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = TodoistStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = TodoistStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time From ef41cd9e35223edf59100f709afa4a78766d597e Mon Sep 17 00:00:00 2001 From: Shivam Date: Mon, 20 Nov 2023 21:14:55 -0800 Subject: [PATCH 2/4] Add sample config details --- .../source-todoist/integration_tests/invalid_config.json | 4 ++-- .../source-todoist/integration_tests/sample_config.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json index f3732995784f..61ab93c78305 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} + "token": "INVALID_API_TOKEN" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json index ecc4913b84c7..654af81febcd 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { - "fix-me": "TODO" -} + "token": "API_TOKEN" +} \ No newline at end of file From e4285ad52f015d1b1a79b53fc83a99142018f44e Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 19 Dec 2023 14:43:51 -0300 Subject: [PATCH 3/4] fix schema --- .../connectors/source-todoist/icon.svg | 14 +++ .../connectors/source-todoist/metadata.yaml | 8 +- .../connectors/source-todoist/setup.py | 3 +- .../source_todoist/manifest.yaml | 4 +- .../source_todoist/schemas/employees.json | 4 +- .../source_todoist/schemas/tasks.json | 85 +++++++++++++++++++ docs/integrations/sources/todoist.md | 3 +- 7 files changed, 110 insertions(+), 11 deletions(-) create mode 100644 airbyte-integrations/connectors/source-todoist/icon.svg create mode 100644 airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json diff --git a/airbyte-integrations/connectors/source-todoist/icon.svg b/airbyte-integrations/connectors/source-todoist/icon.svg new file mode 100644 index 000000000000..c4df54834efe --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/icon.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/metadata.yaml b/airbyte-integrations/connectors/source-todoist/metadata.yaml index d95d9a5696ff..c6fd1c2b58ec 100644 --- a/airbyte-integrations/connectors/source-todoist/metadata.yaml +++ b/airbyte-integrations/connectors/source-todoist/metadata.yaml @@ -1,10 +1,10 @@ data: allowedHosts: hosts: - - TODO # Please change to the hostname of the source. + - api.todoist.com/rest/v2 registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorBuildOptions: @@ -15,13 +15,13 @@ data: connectorSubtype: api connectorType: source definitionId: 1a3d38e4-dc6b-4154-b56b-582f9e978ecd - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-todoist githubIssueLabel: source-todoist icon: todoist.svg license: MIT name: Todoist - releaseDate: TODO + releaseDate: 2023-12-10 releaseStage: alpha supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/todoist diff --git a/airbyte-integrations/connectors/source-todoist/setup.py b/airbyte-integrations/connectors/source-todoist/setup.py index b01fda514b5f..3add92262189 100644 --- a/airbyte-integrations/connectors/source-todoist/setup.py +++ b/airbyte-integrations/connectors/source-todoist/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml index 92eff2ccb219..8460493fe200 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml @@ -36,7 +36,6 @@ definitions: $parameters: path: "/projects" - streams: - "#/definitions/tasks_stream" - "#/definitions/projects_stream" @@ -59,4 +58,5 @@ spec: properties: token: type: string - description: API authorization bearer token for authenticating the API \ No newline at end of file + description: API authorization bearer token for authenticating the API + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json index 16d2acae12c5..c9bce00c9315 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": true, "type": "object", "properties": { @@ -79,4 +79,4 @@ "type": ["null", "string"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json new file mode 100644 index 000000000000..f5d926e1d087 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json @@ -0,0 +1,85 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "assignee_id": { + "type": ["null", "string"] + }, + "assigner_id": { + "type": ["null", "string"] + }, + "comment_count": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "due": { + "anyOf": [ + { + "type": ["null", "object"] + }, + { + "properties": { + "date": { + "type": ["null", "string"] + }, + "is_recurring": { + "type": ["null", "boolean"] + }, + "lang": { + "type": ["null", "string"] + }, + "string": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "is_completed": { + "type": ["null", "boolean"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "order": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "string"] + }, + "section_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/docs/integrations/sources/todoist.md b/docs/integrations/sources/todoist.md index 77935691155c..34578169dabc 100644 --- a/docs/integrations/sources/todoist.md +++ b/docs/integrations/sources/todoist.md @@ -44,4 +44,5 @@ List of available streams: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------| -| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist | +| 0.2.0 | 2023-12-19 | [32690](https://github.com/airbytehq/airbyte/pull/32690) | Migrate to low-code | +| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist | From 9d3511ee49513fc78bd0d1b459072d2f0a653011 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 19 Dec 2023 15:02:43 -0300 Subject: [PATCH 4/4] run format --- .../source-todoist/acceptance-test-config.yml | 28 +++++++++---------- .../integration_tests/configured_catalog.json | 2 +- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../source_todoist/schemas/projects.json | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml index 77f5556cfc2d..25c0dc8dc1d4 100644 --- a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml @@ -19,21 +19,21 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json index 52caaced50a4..8fed59fd4d9e 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json @@ -19,4 +19,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json index 61ab93c78305..244ec5755c74 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { "token": "INVALID_API_TOKEN" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json index 654af81febcd..6dbfca1a0354 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { "token": "API_TOKEN" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json index b10103388dc9..f43f7f3da3f5 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json @@ -40,4 +40,4 @@ "type": ["null", "string"] } } -} \ No newline at end of file +}