From d16883bd62b58704724ae0c3a8ffd59dd1132405 Mon Sep 17 00:00:00 2001 From: Aldre Jota Date: Mon, 16 Oct 2023 01:20:58 +1100 Subject: [PATCH 1/2] Add Manifest from Builder --- .../source_pivotal_tracker/manifest.yaml | 415 ++++++++++++++++++ 1 file changed, 415 insertions(+) create mode 100644 airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/manifest.yaml diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/manifest.yaml b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/manifest.yaml new file mode 100644 index 000000000000..8d0842519d3f --- /dev/null +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/manifest.yaml @@ -0,0 +1,415 @@ +version: 0.50.0 +type: DeclarativeSource + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - "*" + + requester: + type: HttpRequester + url_base: https://www.pivotaltracker.com/services/v5 + http_method: GET + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_token'] }}" + inject_into: + type: RequestOption + field_name: X-TrackerToken + inject_into: header + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + base_properties: + id: + type: number + kind: + type: string + name: + type: string + + base_dates_properties: + $ref: "#/definitions/base_properties" + created_at: + type: string + updated_at: + type: string + + project_entity_properties: + $ref: "#/definitions/base_dates_properties" + project_id: + type: number + + timezone_properties: + properties: + kind: + type: string + offset: + type: string + olson_name: + type: string + type: object + + project_schema: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + $ref: "#/definitions/base_dates_properties" + account_id: + type: number + atom_enabled: + type: boolean + automatic_planning: + type: boolean + bugs_and_chores_are_estimatable: + type: boolean + current_iteration_number: + type: number + enable_following: + type: boolean + enable_incoming_emails: + type: boolean + enable_tasks: + type: boolean + has_google_domain: + type: boolean + initial_velocity: + type: number + iteration_length: + type: number + number_of_done_iterations_to_show: + type: number + point_scale: + type: string + point_scale_is_custom: + type: boolean + project_type: + type: string + public: + type: boolean + show_priority_icon: + type: boolean + show_priority_icon_in_all_panels: + type: boolean + show_story_priority: + type: boolean + start_time: + type: string + time_zone: + $ref: "#/definitions/timezone_properties" + velocity_averaged_over: + type: number + version: + type: number + week_start_day: + type: string + type: object + + projects_stream: + $ref: "#/definitions/base_stream" + name: projects + primary_key: id + schema_loader: + $ref: "#/definitions/project_schema" + $parameters: + path: "/projects" + + project_retriever: + $ref: "#/definitions/retriever" + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: project + stream: + $ref: "#/definitions/projects_stream" + + project_base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/project_retriever" + + epic_schema: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + $ref: "#/definitions/project_entity_properties" + label: + $ref: "#/definitions/project_entity_properties" + url: + type: string + type: object + + epics_stream: + $ref: "#/definitions/project_base_stream" + name: epics + primary_key: id + schema_loader: + $ref: "#/definitions/epic_schema" + $parameters: + path: "/projects/{{ stream_partition.project }}/epics" + + label_schema: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + $ref: "#/definitions/project_entity_properties" + type: object + + labels_stream: + $ref: "#/definitions/project_base_stream" + name: labels + primary_key: id + schema_loader: + $ref: "#/definitions/label_schema" + $parameters: + path: "/projects/{{ stream_partition.project }}/labels" + + person_properties: + type: object + properties: + $ref: "#/definitions/base_properties" + email: + type: string + initials: + type: string + username: + type: string + + membership_schema: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + id: + type: number + kind: + type: string + role: + type: string + person: + $ref: "#/definitions/person_properties" + favorite: + type: boolean + created_at: + type: string + project_id: + type: number + updated_at: + type: string + project_color: + type: string + last_viewed_at: + type: string + wants_comment_notification_emails: + type: boolean + will_receive_mention_notifications_or_emails: + type: boolean + type: object + + memberships_stream: + $ref: "#/definitions/project_base_stream" + name: project_memberships + primary_key: id + schema_loader: + $ref: "#/definitions/membership_schema" + $parameters: + path: "/projects/{{ stream_partition.project }}/memberships" + + project_paginated_retriever: + $ref: "#/definitions/project_retriever" + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: offset + inject_into: request_parameter + pagination_strategy: + type: OffsetIncrement + page_size: 100 + + project_paginated_base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/project_paginated_retriever" + + release_schema: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + $ref: "#/definitions/project_entity_properties" + url: + type: string + labels: + type: array + accepted_at: + type: string + current_state: + type: string + + releases_stream: + $ref: "#/definitions/project_paginated_base_stream" + name: releases + primary_key: id + schema_loader: + $ref: "#/definitions/release_schema" + $parameters: + path: "/projects/{{ stream_partition.project }}/releases" + + story_schema: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + $ref: "#/definitions/project_entity_properties" + url: + type: string + labels: + type: array + items: + type: object + properties: + $ref: "#/definitions/project_entity_properties" + owner_ids: + type: array + story_type: + type: string + accepted_at: + type: string + description: + type: string + current_state: + type: string + story_priority: + type: string + requested_by_id: + type: number + + stories_stream: + $ref: "#/definitions/project_paginated_base_stream" + name: stories + primary_key: id + schema_loader: + $ref: "#/definitions/story_schema" + $parameters: + path: "/projects/{{ stream_partition.project }}/stories" + + activity_schema: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + $ref: "#/definitions/base_dates_properties" + public: + type: boolean + version: + type: number + time_zone: + $ref: "#/definitions/timezone_properties" + account_id: + type: number + start_time: + type: string + point_scale: + type: string + atom_enabled: + type: boolean + enable_tasks: + type: boolean + project_type: + type: string + week_start_day: + type: string + enable_following: + type: boolean + initial_velocity: + type: number + iteration_length: + type: number + has_google_domain: + type: boolean + automatic_planning: + type: boolean + show_priority_icon: + type: boolean + show_story_priority: + type: boolean + point_scale_is_custom: + type: boolean + enable_incoming_emails: + type: boolean + velocity_averaged_over: + type: number + current_iteration_number: + type: number + bugs_and_chores_are_estimatable: + type: boolean + show_priority_icon_in_all_panels: + type: boolean + number_of_done_iterations_to_show: + type: number + + activity_stream: + $ref: "#/definitions/project_paginated_base_stream" + name: activity + primary_key: guid + schema_loader: + $ref: "#/definitions/activity_schema" + $parameters: + path: "/projects/{{ stream_partition.project }}/activity" + +streams: + - "#/definitions/projects_stream" + - "#/definitions/epics_stream" + - "#/definitions/labels_stream" + - "#/definitions/memberships_stream" + - "#/definitions/releases_stream" + - "#/definitions/stories_stream" + - "#/definitions/activity_stream" + +check: + type: CheckStream + stream_names: + - projects + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/pivotal-tracker + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - api_token + properties: + api_token: + type: string + title: API Token + airbyte_secret: true + description: Pivotal Tracker API token + order: 0 + additionalProperties: true From 766bfd5b48da6dc68b6d04cc0cb7263dd94af4ef Mon Sep 17 00:00:00 2001 From: Aldre Jota Date: Mon, 16 Oct 2023 01:23:06 +1100 Subject: [PATCH 2/2] Update Metadata --- .../source-pivotal-tracker/Dockerfile | 2 +- .../source-pivotal-tracker/README.md | 59 +------ .../source-pivotal-tracker/metadata.yaml | 4 +- .../source_pivotal_tracker/source.py | 162 ++---------------- .../source_pivotal_tracker/spec.json | 17 -- 5 files changed, 18 insertions(+), 226 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile b/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile index 016438b17b42..5e2a8130aac3 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile +++ b/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile @@ -34,5 +34,5 @@ COPY source_pivotal_tracker ./source_pivotal_tracker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-pivotal-tracker diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/README.md b/airbyte-integrations/connectors/source-pivotal-tracker/README.md index fcce2540445e..945454a35061 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/README.md +++ b/airbyte-integrations/connectors/source-pivotal-tracker/README.md @@ -1,35 +1,10 @@ # Pivotal Tracker Source -This is the repository for the Pivotal Tracker source connector, written in Python. +This is the repository for the Pivotal Tracker configuration based source connector. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pivotal-tracker). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -47,14 +22,6 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pivotal-tracker test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,30 +46,14 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pivotal-tracker:dev di docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pivotal-tracker:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run:, run ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` To run your integration tests with docker diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml index f3b9c5bb49ef..9bf485eb08ca 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml +++ b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: d60f5393-f99e-4310-8d05-b1876820f40e - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-pivotal-tracker githubIssueLabel: source-pivotal-tracker icon: pivotal-tracker.svg @@ -16,7 +16,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/pivotal-tracker tags: - - language:python + - language:lowcode ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py index 116dbb7c2b86..3b8346029ed1 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py @@ -2,158 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator - -class PivotalTrackerStream(HttpStream, ABC): - - url_base = "https://www.pivotaltracker.com/services/v5/" - primary_key = "id" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - - headers = response.headers - if "X-Tracker-Pagination-Total" not in headers: - return None # not paginating - - page_size = int(headers["X-Tracker-Pagination-Limit"]) - records_returned = int(headers["X-Tracker-Pagination-Returned"]) - current_offset = int(headers["X-Tracker-Pagination-Offset"]) - - if records_returned < page_size: - return None # no more - - return {"offset": current_offset + page_size} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params: MutableMapping[str, Any] = {} - if next_page_token: - params["offset"] = next_page_token["offset"] - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # print(response.json()) - for record in response.json(): # everything is in a list - yield record - - -class Projects(PivotalTrackerStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "projects" - - -class ProjectBasedStream(PivotalTrackerStream): - @property - @abstractmethod - def subpath(self) -> str: - """ - Within the project. For example, "stories" producing: - https://www.pivotaltracker.com/services/v5/projects/{project_id}/stories - """ - - def __init__(self, project_ids: List[str], **kwargs): - super().__init__(**kwargs) - self.project_ids = project_ids - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"projects/{stream_slice['project_id']}/{self.subpath}" - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for project_id in self.project_ids: - yield {"project_id": project_id} - - -class Stories(ProjectBasedStream): - subpath = "stories" - - -class ProjectMemberships(ProjectBasedStream): - subpath = "memberships" - - -class Labels(ProjectBasedStream): - subpath = "labels" - - -class Releases(ProjectBasedStream): - subpath = "releases" - - -class Epics(ProjectBasedStream): - subpath = "epics" - - -class Activity(ProjectBasedStream): - subpath = "activity" - primary_key = "guid" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - for record in super().parse_response(response, **kwargs): - if "project" in record: - record["project_id"] = record["project"]["id"] - yield record - - -# Custom token authenticator because no "Bearer" -class PivotalAuthenticator(HttpAuthenticator): - def __init__(self, token: str): - self._token = token - - def get_auth_header(self) -> Mapping[str, Any]: - return {"X-TrackerToken": self._token} - - -# Source -class SourcePivotalTracker(AbstractSource): - @staticmethod - def _get_authenticator(config: Mapping[str, Any]) -> HttpAuthenticator: - token = config.get("api_token") - return PivotalAuthenticator(token) - - @staticmethod - def _generate_project_ids(auth: HttpAuthenticator) -> List[str]: - """ - Args: - config (dict): Dict representing connector's config - Returns: - List[str]: List of project ids accessible by the api_token - """ - - projects = Projects(authenticator=auth) - records = projects.read_records(SyncMode.full_refresh) - project_ids: List[str] = [] - for record in records: - project_ids.append(record["id"]) - return project_ids - - def check_connection(self, logger, config) -> Tuple[bool, any]: - auth = SourcePivotalTracker._get_authenticator(config) - self._generate_project_ids(auth) - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = self._get_authenticator(config) - project_ids = self._generate_project_ids(auth) - project_args = {"project_ids": project_ids, "authenticator": auth} - return [ - Projects(authenticator=auth), - Stories(**project_args), - ProjectMemberships(**project_args), - Labels(**project_args), - Releases(**project_args), - Epics(**project_args), - Activity(**project_args), - ] +# Declarative Source +class SourcePivotalTracker(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json deleted file mode 100644 index 40eaea438109..000000000000 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pivotal Tracker Spec", - "type": "object", - "required": ["api_token"], - "additionalProperties": false, - "properties": { - "api_token": { - "type": "string", - "description": "Pivotal Tracker API token", - "examples": ["5c054d0de3440452190fdc5d5a04d871"] - } - } - } -}