diff --git a/airbyte-integrations/connectors/source-todoist/.dockerignore b/airbyte-integrations/connectors/source-todoist/.dockerignore deleted file mode 100644 index d853b63208a0..000000000000 --- a/airbyte-integrations/connectors/source-todoist/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_todoist -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-todoist/Dockerfile b/airbyte-integrations/connectors/source-todoist/Dockerfile deleted file mode 100644 index c30309061741..000000000000 --- a/airbyte-integrations/connectors/source-todoist/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_todoist ./source_todoist - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-todoist diff --git a/airbyte-integrations/connectors/source-todoist/README.md b/airbyte-integrations/connectors/source-todoist/README.md index bd25a8590776..b68dcdc0a9bf 100644 --- a/airbyte-integrations/connectors/source-todoist/README.md +++ b/airbyte-integrations/connectors/source-todoist/README.md @@ -1,37 +1,13 @@ # Todoist Source -This is the repository for the Todoist source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/todoist). +This is the repository for the Todoist configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/todoist). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/todoist) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/todoist) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_todoist/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -39,29 +15,69 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source todoist test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -#### Build -**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name=source-todoist build +airbyte-ci connectors --name source-todoist build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-todoist:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -An image will be built with the tag `airbyte/source-todoist:dev`. +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. -**Via `docker build`:** +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-todoist:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: ```bash docker build -t airbyte/source-todoist:dev . -``` +# Running the spec command against your patched connector +docker run airbyte/source-todoist:dev spec #### Run Then run any of the connector commands as follows: @@ -71,16 +87,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-todoist:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-todoist:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-todoist:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` - ## Testing -You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -```bash -airbyte-ci connectors --name=source-todoist test -``` -### Customizing acceptance Tests +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-todoist test +``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connectors/source-todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml index b14d04a96904..25c0dc8dc1d4 100644 --- a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml @@ -1,7 +1,6 @@ -# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-todoist:dev -test_strictness_level: low acceptance_tests: spec: tests: @@ -22,7 +21,7 @@ acceptance_tests: empty_streams: [] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: - # path: "integration_tests/expected_records.txt" + # path: "integration_tests/expected_records.jsonl" # extra_fields: no # exact_order: no # extra_records: yes @@ -32,7 +31,8 @@ acceptance_tests: # tests: # - config_path: "secrets/config.json" # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-todoist/icon.svg b/airbyte-integrations/connectors/source-todoist/icon.svg index dbf417c99c8d..c4df54834efe 100644 --- a/airbyte-integrations/connectors/source-todoist/icon.svg +++ b/airbyte-integrations/connectors/source-todoist/icon.svg @@ -1,6 +1,14 @@ - - - - - - + + + + + + + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py index 1100c1c58cf5..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json index 09f16c3ccf2a..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json @@ -1,37 +1,5 @@ { - "tasks": { - "id": false, - "project_id": 10, - "section_id": -10, - "content": 10, - "description": true, - "is_completed": "not so true", - "labels": [true, false], - "parent_id": -50, - "order": true, - "priority": "10", - "due": true, - "url": 50, - "comment_count": "10", - "created_at": { "when": true }, - "creator_id": -1, - "assignee_id": 20, - "assigner_id": 50 - }, - "projects": { - "id": { - "number": "2203306141" - }, - "name": 50, - "comment_count": false, - "order": "1", - "color": 100, - "is_shared": [false], - "is_favorite": "world", - "parent_id": 100, - "is_inbox_project": [true], - "is_team_inbox": "false", - "view_style": ["list"], - "url": ["https://todoist.com/showProject?id=2203306141"] + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json index 275f28ac14f5..8fed59fd4d9e 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json @@ -1,34 +1,22 @@ { "streams": [ { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "tasks", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "projects", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json index 37cbe64ae615..244ec5755c74 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "token": "INVALID TOKEN" + "token": "INVALID_API_TOKEN" } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json index 55e640b7e5c3..6dbfca1a0354 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { - "token": "VALID TOKEN" + "token": "API_TOKEN" } diff --git a/airbyte-integrations/connectors/source-todoist/metadata.yaml b/airbyte-integrations/connectors/source-todoist/metadata.yaml index f32560daec66..c6fd1c2b58ec 100644 --- a/airbyte-integrations/connectors/source-todoist/metadata.yaml +++ b/airbyte-integrations/connectors/source-todoist/metadata.yaml @@ -1,24 +1,30 @@ data: + allowedHosts: + hosts: + - api.todoist.com/rest/v2 + registries: + oss: + enabled: true + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source - definitionId: 7d272065-c316-4c04-a433-cd4ee143f83e - dockerImageTag: 0.1.0 + definitionId: 1a3d38e4-dc6b-4154-b56b-582f9e978ecd + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-todoist githubIssueLabel: source-todoist icon: todoist.svg license: MIT name: Todoist - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-12-10 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/todoist tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-todoist/requirements.txt b/airbyte-integrations/connectors/source-todoist/requirements.txt index ecf975e2fa63..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-todoist/requirements.txt +++ b/airbyte-integrations/connectors/source-todoist/requirements.txt @@ -1 +1 @@ --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-todoist/setup.py b/airbyte-integrations/connectors/source-todoist/setup.py index cf672aeb4515..3add92262189 100644 --- a/airbyte-integrations/connectors/source-todoist/setup.py +++ b/airbyte-integrations/connectors/source-todoist/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6", - "requests_mock~=1.8", + "pytest~=6.2", + "pytest-mock~=3.6.1", ] setup( diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py index f04f17ce4a6f..62d0c357fa93 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml new file mode 100644 index 000000000000..8460493fe200 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml @@ -0,0 +1,62 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.todoist.com/rest/v2" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + tasks_stream: + $ref: "#/definitions/base_stream" + name: "tasks" + $parameters: + path: "/tasks" + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + $parameters: + path: "/projects" + +streams: + - "#/definitions/tasks_stream" + - "#/definitions/projects_stream" + +check: + type: CheckStream + stream_names: + - "tasks" + - "projects" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/source-todolist + connection_specification: + title: Source Todolist Spec + type: object + required: + - token + additionalProperties: true + properties: + token: + type: string + description: API authorization bearer token for authenticating the API + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json new file mode 100644 index 000000000000..c9bce00c9315 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json @@ -0,0 +1,82 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "assignee_id": { + "type": ["null", "string"] + }, + "assigner_id": { + "type": ["null", "string"] + }, + "comment_count": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "due": { + "anyOf": [ + { + "type": ["null", "object"] + }, + { + "properties": { + "date": { + "type": ["null", "string"] + }, + "is_recurring": { + "type": ["null", "boolean"] + }, + "lang": { + "type": ["null", "string"] + }, + "string": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "is_completed": { + "type": ["null", "boolean"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "order": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "string"] + }, + "section_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json index cf22f8da8a4b..f5d926e1d087 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": true, "type": "object", "properties": { @@ -51,6 +51,9 @@ "id": { "type": ["null", "string"] }, + "duration": { + "type": ["null", "string"] + }, "is_completed": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py index 517c10ae7383..b363f5fb2623 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py @@ -2,66 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class TodoistStream(HttpStream): - """ - Stream for Todoist REST API : https://developer.todoist.com/rest/v2/#overview - """ - @property - def url_base(self) -> str: - return "https://api.todoist.com/rest/v2/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.name.title().lower() - - -class Tasks(TodoistStream): - - primary_key = "id" - - -class Projects(TodoistStream): - - primary_key = "id" - - -# Source -class SourceTodoist(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config["token"] - authenticator = TokenAuthenticator(token=token) - task_stream = Tasks(authenticator) - task_records = task_stream.read_records(sync_mode="full_refresh") - next(task_records) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - token = config["token"] - auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support - return [Tasks(authenticator=auth), Projects(authenticator=auth)] +# Declarative Source +class SourceTodoist(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml deleted file mode 100644 index 19e7f9dc2e42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -documentationUrl: https://docs.airbyte.io/integrations/sources/todoist -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Todoist Spec - type: object - required: - - token - properties: - token: - type: string - description: >- - Your API Token. See here. The token is - case sensitive. - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py deleted file mode 100644 index d4dbafa573be..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_todoist.source import SourceTodoist - - -def test_check_connection(mocker): - source = SourceTodoist() - fake_info_record = {"collection": "is_mocked"} - with patch("source_todoist.source.Tasks.read_records", MagicMock(return_value=iter([fake_info_record]))): - logger_mock = MagicMock() - config_mock = {"token": "test"} - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceTodoist() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py deleted file mode 100644 index af1ed528da42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_todoist.source import TodoistStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(TodoistStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = TodoistStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = TodoistStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = TodoistStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = TodoistStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/todoist.md b/docs/integrations/sources/todoist.md index 77935691155c..34578169dabc 100644 --- a/docs/integrations/sources/todoist.md +++ b/docs/integrations/sources/todoist.md @@ -44,4 +44,5 @@ List of available streams: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------| -| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist | +| 0.2.0 | 2023-12-19 | [32690](https://github.com/airbytehq/airbyte/pull/32690) | Migrate to low-code | +| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist |