diff --git a/airbyte-integrations/connectors/source-pipedrive/Dockerfile b/airbyte-integrations/connectors/source-pipedrive/Dockerfile index 9ef2f33510dd..1a10b2a1c8f6 100644 --- a/airbyte-integrations/connectors/source-pipedrive/Dockerfile +++ b/airbyte-integrations/connectors/source-pipedrive/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_pipedrive ./source_pipedrive + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_pipedrive ./source_pipedrive ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-pipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/README.md b/airbyte-integrations/connectors/source-pipedrive/README.md index 25eaff686dfa..e61b021a22f5 100644 --- a/airbyte-integrations/connectors/source-pipedrive/README.md +++ b/airbyte-integrations/connectors/source-pipedrive/README.md @@ -1,34 +1,10 @@ # Pipedrive Source -This is the repository for the Pipedrive source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pipedrive). +This is the repository for the Pipedrive configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/pipedrive). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install . -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pipedrive) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pipedrive/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/pipedrive) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pipedrive/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pipedrive test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pipedrive:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pipedrive:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-pipedrive/__init__.py similarity index 61% rename from airbyte-integrations/connectors/source-pipedrive/integration_tests/integration_test.py rename to airbyte-integrations/connectors/source-pipedrive/__init__.py index 5544269fad67..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-pipedrive/__init__.py @@ -1,7 +1,3 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - - -def test_example(): - assert True diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml index 928821e53d54..0314b069299b 100644 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml @@ -1,67 +1,36 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-pipedrive:dev -test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_pipedrive/spec.json" + - spec_path: "source_pipedrive/spec.yaml" connection: tests: - config_path: "secrets/config.json" status: "succeed" - - config_path: "secrets/old_config.json" - status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: tests: - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: 0.1.19 basic_read: tests: - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - ignored_fields: - users: - - name: modified - bypass_reason: "constantly increasing date-time field" - - name: last_login - bypass_reason: "constantly increasing date-time field" - deal_fields: - - name: show_in_pipelines - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: pipeline_ids - bypass_reason: "Unstable data" - - name: update_time - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" - organization_fields: - - name: update_time - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" - person_fields: - - name: update_time - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" - product_fields: - - name: update_time - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" + configured_catalog_path: "integration_tests/configured_catalog.json" fail_on_extra_columns: false - incremental: + empty_streams: + - name: files + - name: filters + - name: leads + - name: notes + - name: activities + - name: pipelines + - name: products + - name: stages + - name: deal_products + + incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json index 4ff5d5327d40..afa3150857dd 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json @@ -4,24 +4,22 @@ "stream": { "name": "deals", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh","incremental"], "source_defined_cursor": true, "default_cursor_field": ["update_time"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "cursor_field": ["update_time"], "destination_sync_mode": "append" }, { "stream": { - "name": "deal_products", + "name": "deal_fields", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { @@ -87,61 +85,60 @@ }, { "stream": { - "name": "activity_fields", + "name": "activities", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" }, { "stream": { "name": "activity_types", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["update_time"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "activities", + "name": "activity_fields", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { "name": "currencies", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { "name": "organizations", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "organization_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -166,6 +163,15 @@ "cursor_field": ["update_time"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "person_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "pipelines", @@ -178,17 +184,6 @@ "cursor_field": ["update_time"], "destination_sync_mode": "append" }, - { - "stream": { - "name": "product_fields", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, { "stream": { "name": "products", @@ -205,74 +200,58 @@ }, { "stream": { - "name": "roles", + "name": "product_fields", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, - { - "stream": { - "name": "stages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] - }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "users", + "name": "roles", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["modified"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["modified"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "deal_fields", + "name": "stages", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "default_cursor_field": ["update_time"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", + "destination_sync_mode": "overwrite", "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "primary_key": [["id"]] }, { "stream": { - "name": "organization_fields", + "name": "users", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "default_cursor_field": ["modified"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "destination_sync_mode": "overwrite", + "cursor_field": ["modified"], + "primary_key": [["id"]] }, { "stream": { - "name": "person_fields", + "name": "deal_products", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl deleted file mode 100644 index c593d1cc0ba5..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl +++ /dev/null @@ -1,62 +0,0 @@ -{"stream": "activities", "data": {"id": 1, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "task", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-06", "due_time": "", "duration": "", "busy_flag": true, "add_time": "2021-07-06 15:02:04", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Task1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 10, "lead_id": null, "active_flag": true, "update_time": "2021-07-06 15:02:03", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy)", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal10@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Task", "lead": null}, "emitted_at": 1690801204738} -{"stream": "activities", "data": {"id": 3, "company_id": 7780468, "user_id": 11884360, "done": true, "type": "deadline", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-28", "due_time": "15:15", "duration": "00:30", "busy_flag": true, "add_time": "2021-07-06 15:03:31", "marked_as_done_time": "2023-02-22 08:25:45", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Deadline1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 1, "lead_id": null, "active_flag": true, "update_time": "2023-02-22 08:25:49", "update_user_id": 11884360, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "New note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal1@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Deadline", "lead": null}, "emitted_at": 1690801204739} -{"stream": "activities", "data": {"id": 7, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "call", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2023-02-22", "due_time": "12:30", "duration": "01:00", "busy_flag": true, "add_time": "2023-02-22 08:52:52", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": null, "subject": "Call", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 3, "person_id": 7, "deal_id": null, "lead_id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "active_flag": true, "update_time": "2023-02-22 08:52:52", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Test call
", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 7, "primary_flag": true}, {"person_id": 9, "primary_flag": false}], "series": null, "is_recurring": null, "org_name": "Test Organization 3", "person_name": "User6 Sample", "deal_title": null, "lead_title": "Test Organization 3 lead", "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": null, "assigned_to_user_id": 11884360, "type_name": "Call", "lead": {"id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "title": "Test Organization 3 lead", "labels": "aecece60-c069-11eb-93bf-b59c4f1731e6", "source": "Manually created", "owner_id": 11884360, "creator_user_id": 11884360, "deal_id": null, "related_person_id": 4, "related_org_id": 2, "person_name": null, "person_phone": null, "person_email": null, "org_name": null, "org_address": null, "active_flag": true, "add_time": "2023-02-22 08:52:06.047", "update_time": "2023-02-22 11:44:31.718", "archive_time": null, "seen": true, "deal_value": 1200, "deal_currency": "USD", "next_activity_id": 7, "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "visible_to": 3, "source_reference_id": null, "user": null, "deal_expected_close_date": "2023-05-15", "next_activity_status": null, "next_activity_datetime": null, "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null}}, "emitted_at": 1690801204739} -{"stream": "activity_fields", "data": {"id": 1, "key": "id", "name": "ID", "order_nr": 1, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205420} -{"stream": "activity_fields", "data": {"id": 2, "key": "subject", "name": "Subject", "order_nr": 2, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205420} -{"stream": "activity_fields", "data": {"id": 3, "key": "type", "name": "Type", "order_nr": 3, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "options": [{"id": "call", "label": "Call"}, {"id": "meeting", "label": "Meeting"}, {"id": "task", "label": "Task"}, {"id": "deadline", "label": "Deadline"}, {"id": "email", "label": "Email"}, {"id": "lunch", "label": "Lunch"}, {"id": "test_1", "label": "Test 1"}], "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205421} -{"stream": "activity_types", "data": {"id": 7, "order_nr": 7, "name": "Test 1", "key_string": "test_1", "icon_key": "car", "active_flag": true, "color": null, "is_custom_flag": true, "add_time": "2023-02-22 11:13:54", "update_time": "2023-02-22 11:13:54"}, "emitted_at": 1690801206161} -{"stream": "currencies", "data": {"id": 2, "code": "AFN", "name": "Afghanistan Afghani", "symbol": "AFN", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206834} -{"stream": "currencies", "data": {"id": 3, "code": "ALL", "name": "Albanian Lek", "symbol": "ALL", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206834} -{"stream": "currencies", "data": {"id": 41, "code": "DZD", "name": "Algerian Dinar", "symbol": "DZD", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206835} -{"stream": "deals", "data": {"id": 11, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 10, "org_id": 7, "stage_id": 1, "title": "Test Organization 2 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:27:53", "update_time": "2023-02-22 08:27:54", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-02-28", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User9 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal11@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207614} -{"stream": "deals", "data": {"id": 13, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 8, "org_id": 7, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1200, "currency": "USD", "add_time": "2023-02-22 08:53:17", "update_time": "2023-02-22 08:53:18", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-03-31", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User7 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,200", "weighted_value": 1200, "formatted_weighted_value": "$1,200", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal13@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207615} -{"stream": "deals", "data": {"id": 14, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 5, "org_id": 1, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:54:48", "update_time": "2023-02-22 08:54:49", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-04-30", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User2 Sample", "org_name": "Test Organization1", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal14@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207615} -{"stream": "deal_products", "data": {"id": 11, "deal_id": 17, "product_id": 2, "product_variation_id": null, "name": "Item 1", "order_nr": 1, "item_price": 100, "quantity": 6, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 600, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 08:59:11", "last_edit": "2023-02-22 08:59:11", "comments": null, "tax": 0, "quantity_formatted": "6", "sum_formatted": "$600", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801209362} -{"stream": "deal_products", "data": {"id": 12, "deal_id": 20, "product_id": 7, "product_variation_id": null, "name": "Item 6", "order_nr": 1, "item_price": 100, "quantity": 30, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 3000, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:24:57", "last_edit": "2023-02-22 10:24:57", "comments": null, "tax": 0, "quantity_formatted": "30", "sum_formatted": "$3,000", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801209809} -{"stream": "deal_products", "data": {"id": 15, "deal_id": 23, "product_id": 11, "product_variation_id": null, "name": "Item 10", "order_nr": 1, "item_price": 20, "quantity": 120, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 2400, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:30:01", "last_edit": "2023-02-22 10:30:01", "comments": null, "tax": 0.01, "quantity_formatted": "120", "sum_formatted": "$2,400", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801210358} -{"stream": "deal_fields", "data": {"id": 12477, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1690801212684} -{"stream": "deal_fields", "data": {"id": 12453, "key": "title", "name": "Title", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:02", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "use_field": "id", "link": "/deal/", "mandatory_flag": true}, "emitted_at": 1690801212685} -{"stream": "deal_fields", "data": {"id": 12454, "key": "creator_user_id", "name": "Creator", "group_id": null, "order_nr": 0, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1690801212685} -{"stream": "files", "data": {"id": 1, "user_id": 11884360, "log_id": null, "add_time": "2023-02-22 10:52:23", "update_time": "2023-02-22 10:52:23", "file_name": "bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "file_size": 6132, "active_flag": true, "inline_flag": false, "remote_location": "s3", "remote_id": "company/7780468/user/11884360/files/bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "s3_bucket": null, "url": "https://app.pipedrive.com/api/v1/files/1/download", "name": "Airbyte logo 75x75.png", "description": null, "deal_id": null, "lead_id": null, "person_id": null, "org_id": 1, "product_id": null, "activity_id": null, "deal_name": null, "lead_name": null, "person_name": null, "people_name": null, "org_name": "Test Organization1", "product_name": null, "mail_message_id": null, "mail_template_id": null, "cid": null, "file_type": "img"}, "emitted_at": 1690801213534} -{"stream": "filters", "data": {"id": 29, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1690801214182} -{"stream": "filters", "data": {"id": 28, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1690801214183} -{"stream": "lead_labels", "data": {"id": "aecece60-c069-11eb-93bf-b59c4f1731e6", "name": "Hot", "color": "red", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214865} -{"stream": "lead_labels", "data": {"id": "aecece61-c069-11eb-93bf-b59c4f1731e6", "name": "Warm", "color": "yellow", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214866} -{"stream": "lead_labels", "data": {"id": "aecece62-c069-11eb-93bf-b59c4f1731e6", "name": "Cold", "color": "blue", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214866} -{"stream": "leads", "data": {"id": "7220c7f0-de6b-11eb-a544-5391b24dc0cf", "title": "Airbyte1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 10000, "currency": "USD"}, "expected_close_date": "2023-05-16", "person_id": 2, "organization_id": null, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 24, "add_time": "2021-07-06T15:04:22.255Z", "update_time": "2023-02-22T11:46:49.844Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadf6oUYVtZxhWEEyLegJqsRe@pipedrivemail.com"}, "emitted_at": 1690801215787} -{"stream": "leads", "data": {"id": "918d8510-de6b-11eb-8042-15d11dd01d20", "title": "Test Organization1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece61-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 3333, "currency": "USD"}, "expected_close_date": "2023-04-29", "person_id": 3, "organization_id": 1, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 25, "add_time": "2021-07-06T15:05:14.977Z", "update_time": "2023-02-22T11:47:18.002Z", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": 10, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": "Test", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadiYsMXTdfCQEKU896kY5xDA@pipedrivemail.com"}, "emitted_at": 1690801215788} -{"stream": "leads", "data": {"id": "bcfcf9c0-b28a-11ed-a6b1-df90b19da35a", "title": "Test Organization 2 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 15000, "currency": "USD"}, "expected_close_date": "2023-04-06", "person_id": 9, "organization_id": 9, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 26, "add_time": "2023-02-22T08:27:26.428Z", "update_time": "2023-02-22T11:47:36.779Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadpkxTNcD2sbWtbcQKvFKfu3@pipedrivemail.com"}, "emitted_at": 1690801215788} -{"stream": "notes", "data": {"id": 1, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 1, "lead_id": null, "content": "Test Note 1
", "add_time": "2023-02-22 08:24:35", "update_time": "2023-02-22 08:24:35", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization1"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216928} -{"stream": "notes", "data": {"id": 2, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 2, "lead_id": null, "content": "Test note 2
", "add_time": "2023-02-22 08:31:47", "update_time": "2023-02-22 08:31:47", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 2"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216929} -{"stream": "notes", "data": {"id": 3, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 5, "lead_id": null, "content": "Test Note
", "add_time": "2023-02-22 08:59:20", "update_time": "2023-02-22 08:59:20", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 5"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216929} -{"stream": "organizations", "data": {"id": 12, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 7", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:21:58", "delete_time": null, "add_time": "2023-02-22 08:18:16", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "DY Patil College, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Pimpri Colony", "address_locality": "Pimpri-Chinchwad", "address_admin_area_level_1": "Maharashtra", "address_admin_area_level_2": "Pune Division", "address_country": "India", "address_postal_code": "411018", "address_formatted_address": "DY Patil College, DR. D Y PATIL MEDICAL COLLEGE, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra 411018, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217744} -{"stream": "organizations", "data": {"id": 13, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 6", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:23:17", "delete_time": null, "add_time": "2023-02-22 08:18:33", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "Anand Vihar Railway Station, Block D, Anand Vihar, Delhi, Uttar Pradesh, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Anand Vihar", "address_locality": "Delhi", "address_admin_area_level_1": "Uttar Pradesh", "address_admin_area_level_2": "Delhi Division", "address_country": "India", "address_postal_code": "261205", "address_formatted_address": "J8X8+F33, Block D, Anand Vihar, Delhi, Uttar Pradesh 261205, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217746} -{"stream": "organizations", "data": {"id": 3, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 3", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 2, "done_activities_count": 0, "undone_activities_count": 2, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:56:24", "delete_time": null, "add_time": "2023-02-22 08:07:28", "visible_to": "3", "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "next_activity_id": 7, "last_activity_id": null, "last_activity_date": null, "label": 6, "address": "Strasbourg, France", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "", "address_locality": "Strasbourg", "address_admin_area_level_1": "Grand Est", "address_admin_area_level_2": "Bas-Rhin", "address_country": "France", "address_postal_code": "", "address_formatted_address": "Strasbourg, France", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217746} -{"stream": "organization_fields", "data": {"id": 4012, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801218387} -{"stream": "organization_fields", "data": {"id": 4002, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:05", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/organization/", "mandatory_flag": true}, "emitted_at": 1690801218387} -{"stream": "organization_fields", "data": {"id": 4003, "key": "label", "name": "Label", "group_id": null, "order_nr": 0, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:06", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "options": [{"id": 5, "label": "Customer", "color": "green"}, {"id": 6, "label": "Hot lead", "color": "red"}, {"id": 7, "label": "Warm lead", "color": "yellow"}, {"id": 8, "label": "Cold lead", "color": "blue"}], "mandatory_flag": false}, "emitted_at": 1690801218388} -{"stream": "permission_sets", "data": {"id": "79b42bf0-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals admin", "assignment_count": 1, "app": "sales", "type": "admin"}, "emitted_at": 1690801219068} -{"stream": "permission_sets", "data": {"id": "79b42bf1-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals regular user", "assignment_count": 4, "app": "sales", "type": "regular"}, "emitted_at": 1690801219069} -{"stream": "permission_sets", "data": {"id": "fa17fff0-db56-11ec-93f1-e9cfc58fcd59", "name": "Global admin", "assignment_count": 1, "app": "global", "type": "admin"}, "emitted_at": 1690801219069} -{"stream": "persons", "data": {"id": 5, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User2 Sample", "first_name": "User2", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+14443332266", "primary": true}], "email": [{"label": "work", "value": "user2.sample.airbyte@gmail.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:44:49", "delete_time": null, "add_time": "2023-02-22 08:13:09", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 3, "picture_128_url": null, "org_name": "Test Organization1", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219881} -{"stream": "persons", "data": {"id": 6, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User3 Sample", "first_name": "User3", "last_name": "Sample", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+15554442233", "primary": true}], "email": [{"label": "work", "value": "user3.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:45:52", "delete_time": null, "add_time": "2023-02-22 08:13:53", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 2, "picture_128_url": null, "org_name": "Test Organization1", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219883} -{"stream": "persons", "data": {"id": 2, "company_id": 7780468, "owner_id": 11884360, "org_id": 11, "name": "User4 Sample", "first_name": "User4", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 1, "done_activities_count": 0, "undone_activities_count": 1, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+16665552233", "primary": true}], "email": [{"label": "work", "value": "user4.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 11:46:49", "delete_time": null, "add_time": "2021-07-06 15:04:21", "visible_to": "3", "picture_id": null, "next_activity_date": "2023-03-15", "next_activity_time": "11:30:00", "next_activity_id": 24, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 1, "picture_128_url": null, "org_name": "Test Organization 4", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219883} -{"stream": "person_fields", "data": {"id": 9051, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801220805} -{"stream": "person_fields", "data": {"id": 9039, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:04", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/person/", "mandatory_flag": true}, "emitted_at": 1690801220806} -{"stream": "person_fields", "data": {"id": 9040, "key": "label", "name": "Label", "group_id": null, "order_nr": 0, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:04", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "options": [{"id": 1, "label": "Customer", "color": "green"}, {"id": 2, "label": "Hot lead", "color": "red"}, {"id": 3, "label": "Warm lead", "color": "yellow"}, {"id": 4, "label": "Cold lead", "color": "blue"}], "mandatory_flag": false}, "emitted_at": 1690801220806} -{"stream": "pipelines", "data": {"id": 2, "name": "New pipeline", "url_title": "New-pipeline", "order_nr": 2, "active": true, "deal_probability": true, "add_time": "2021-07-06 15:24:11", "update_time": "2021-07-06 15:24:11"}, "emitted_at": 1690801221622} -{"stream": "pipelines", "data": {"id": 3, "name": "New pipeline 2", "url_title": "New-pipeline-2", "order_nr": 3, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:42:31", "update_time": "2023-02-22 10:42:31"}, "emitted_at": 1690801221623} -{"stream": "pipelines", "data": {"id": 4, "name": "New pipeline 3", "url_title": "New-pipeline-3", "order_nr": 4, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:43:12", "update_time": "2023-02-22 10:43:12"}, "emitted_at": 1690801221623} -{"stream": "product_fields", "data": {"id": 23, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801222343} -{"stream": "product_fields", "data": {"id": 17, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:07", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/api/v1/products/", "mandatory_flag": true}, "emitted_at": 1690801222345} -{"stream": "product_fields", "data": {"id": 18, "key": "owner_id", "name": "Owner", "group_id": null, "order_nr": 0, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:07", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "owner_id", "display_field": "owner_name", "mandatory_flag": true}, "emitted_at": 1690801222345} -{"stream": "products", "data": {"id": 1, "name": "Test Product", "code": "12345", "description": null, "unit": "Kg", "tax": 5, "category": "12", "active_flag": true, "selectable": true, "first_char": "t", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2021-11-09 11:32:16", "update_time": "2021-11-09 11:32:31", "prices": [{"id": 1, "product_id": 1, "price": 10, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223125} -{"stream": "products", "data": {"id": 2, "name": "Item 1", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2023-02-22 08:31:13", "update_time": "2023-02-22 08:31:13", "prices": [{"id": 2, "product_id": 2, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223126} -{"stream": "products", "data": {"id": 3, "name": "Item 2", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2023-02-22 08:31:14", "update_time": "2023-02-22 08:31:14", "prices": [{"id": 3, "product_id": 3, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223126} -{"stream": "roles", "data": {"id": 1, "parent_role_id": null, "name": "(Unassigned users)", "active_flag": true, "assignment_count": "5", "sub_role_count": "0", "level": 1, "description": "This is the default group for managing your visibility settings. New users are added automatically unless you change their group when you invite them."}, "emitted_at": 1690801224387} -{"stream": "stages", "data": {"id": 11, "order_nr": 5, "name": "Negotiations Started", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} -{"stream": "stages", "data": {"id": 10, "order_nr": 4, "name": "Proposal Made", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} -{"stream": "stages", "data": {"id": 9, "order_nr": 3, "name": "Demo Scheduled", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} -{"stream": "users", "data": {"id": 18276123, "name": "User3 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user3.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:43", "modified": "2023-03-22 14:34:12", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:34:12", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226031} -{"stream": "users", "data": {"id": 18276134, "name": "User4 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user4.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:44", "modified": "2023-03-22 14:39:38", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:39:38", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226032} -{"stream": "users", "data": {"id": 18276145, "name": "User5 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user5.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:44", "modified": "2023-03-22 14:44:35", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:44:35", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226032} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json index 44d3a2be1b95..7831cc4648f2 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json @@ -1,7 +1,4 @@ { - "authorization": { - "auth_type": "Token", - "api_token": "wrong-api-token" - }, - "replication_start_date": "2021-01-01T10:10:10Z" + "api_key": "api_key", + "replication_start_date": "2017-01-25 00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json index 5d3a28f22a86..7831cc4648f2 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { - "api_token": "", - "replication_start_date": "2021-06-01T10:10:10Z" + "api_key": "api_key", + "replication_start_date": "2017-01-25 00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json index e4a812b80467..d7ca6e9987cf 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json @@ -1,68 +1,35 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "deals" - } - } +{ + "deals": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "activities" - } - } + "files": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "persons" - } - } + "filters": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "pipelines" - } - } + "notes": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "stages" - } - } + "activites": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "modified": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "users" - } - } + "oragnizations": { + "update_time": "2017-06-01 10:10:10" + }, + "persons": { + "update_time": "2017-06-01 10:10:10" + }, + "pipelines": { + "update_time": "2017-06-01 10:10:10" + }, + "products": { + "update_time": "2017-06-01 10:10:10" + }, + "stages": { + "update_time": "2017-06-01 10:10:10" + }, + "users": { + "update_time": "2017-06-01 10:10:10" } -] +} diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index d8a469a9adb3..b9bf6bb80b17 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -4,24 +4,30 @@ data: sl: 100 allowedHosts: hosts: - - api.pipedrive.com + - api.pipedrive.com # Please change to the hostname of the source. + registries: + cloud: + enabled: true + oss: + enabled: true connectorSubtype: api connectorType: source definitionId: d8286229-c680-4063-8c59-23b9b391c700 - dockerImageTag: 1.0.0 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-pipedrive documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive githubIssueLabel: source-pipedrive icon: pipedrive.svg license: MIT name: Pipedrive - registries: - cloud: - enabled: true - oss: - enabled: true - releaseStage: beta + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-10-04 + message: "This version removes the `pipeline_ids` field from the `deal_fields` stream. Config has changed to only use API key. Please update your config." + releaseDate: 2021-07-19 + releaseStage: alpha supportLevel: community tags: - - language:python + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pipedrive/setup.py b/airbyte-integrations/connectors/source-pipedrive/setup.py index 7108dd89b957..7e72591c8797 100644 --- a/airbyte-integrations/connectors/source-pipedrive/setup.py +++ b/airbyte-integrations/connectors/source-pipedrive/setup.py @@ -6,12 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", - "pendulum~=2.1", - "requests~=2.25", + "airbyte-cdk", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", +] setup( name="source_pipedrive", @@ -20,7 +21,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py index 7e21a5c1b775..a2d5a962fa36 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py @@ -1,26 +1,7 @@ -""" -MIT License +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" from .source import SourcePipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/auth.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/auth.py deleted file mode 100644 index 97a7a9ae6081..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/auth.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from requests.auth import AuthBase - - -class QueryStringTokenAuthenticator(AuthBase): - """ - Authenticator that attaches a set of query string parameters (e.g. an API key) to the request. - """ - - def __init__(self, **kwargs): - self.params = kwargs - - def __call__(self, request): - if self.params: - request.prepare_url(request.url, self.params) - return request diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml new file mode 100644 index 000000000000..42240785560b --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml @@ -0,0 +1,359 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + + selector_increment: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "*", "data"] + + selector_users: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "*", "data","*"] + + requester: + type: HttpRequester + url_base: "https://api.pipedrive.com/v1" + http_method: "GET" + request_parameters: + api_token: "{{ config['api_token'] }}" + limit: "50" + items: "{{ parameters.path_extractor }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response['additional_data']['pagination']['next_start'] }}" + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "start" + requester: + $ref: "#/definitions/requester" + + retriever_increment: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_increment" + + incremental_base: + type: DatetimeBasedCursor + cursor_field: "update_time" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + datetime: "{{ format_datetime(config['replication_start_date'], '%Y-%m-%d %H:%M:%S') }}" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_time_option: + field_name: "since_timestamp" + inject_into: "request_parameter" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + deals_stream: + $ref: "#/definitions/base_stream" + name: "deals" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "deal" + path: "/recents" + + deal_fields_stream: + $ref: "#/definitions/base_stream" + name: "deal_fields" + $parameters: + path: "/dealFields" + + files_stream: + $ref: "#/definitions/base_stream" + name: "files" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "file" + path: "/recents" + + filters_stream: + $ref: "#/definitions/base_stream" + name: "filters" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "filter" + path: "/recents" + + lead_labels_stream: + # Does not support pagination + $ref: "#/definitions/base_stream" + name: "lead_labels" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/leadLabels" + + leads_stream: + $ref: "#/definitions/base_stream" + name: "leads" + $parameters: + path: "/leads" + + notes_stream: + $ref: "#/definitions/base_stream" + name: "notes" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "note" + path: "/recents" + + activities_stream: + $ref: "#/definitions/base_stream" + name: "activities" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "activity" + path: "/recents" + + activity_types_stream: +# This stream didnt have limit as query_parameter + $ref: "#/definitions/base_stream" + name: "activity_types" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/activityTypes" + + activity_fields_stream: + $ref: "#/definitions/base_stream" + name: "activity_fields" + $parameters: + path: "/activityFields" + + currencies_stream: + $ref: "#/definitions/base_stream" + name: "currencies" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/currencies" + + organization_stream: + $ref: "#/definitions/base_stream" + name: "organizations" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + $parameters: + path_extractor: "organization" + path: "/organizations" + + organization_fields_stream: + $ref: "#/definitions/base_stream" + name: "organization_fields" + $parameters: + path: "/organizationFields" + + permission_sets_stream: + $ref: "#/definitions/base_stream" + name: "permission_sets" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/permissionSets" + + persons_stream: + $ref: "#/definitions/base_stream" + name: "persons" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "person" + path: "/recents" + + person_fields_stream: + $ref: "#/definitions/base_stream" + name: "person_fields" + $parameters: + path: "/personFields" + + pipelines_stream: + $ref: "#/definitions/base_stream" + name: "pipelines" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "pipeline" + path: "/recents" + + products_stream: + $ref: "#/definitions/base_stream" + name: "products" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "product" + path: "/recents" + + product_fields_stream: + $ref: "#/definitions/base_stream" + name: "product_fields" + $parameters: + path: "/productFields" + + roles_stream: + $ref: "#/definitions/base_stream" + name: "roles" + $parameters: + path: "/roles" + + stages_stream: + $ref: "#/definitions/base_stream" + name: "stages" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "stage" + path: "/recents" + + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_users" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: "modified" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + datetime: "{{ format_datetime(config['replication_start_date'], '%Y-%m-%d %H:%M:%S') }}" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_time_option: + field_name: "since_timestamp" + inject_into: "request_parameter" + $parameters: + path_extractor: "user" + path: "/recents" + + deal_products_stream: + name: "deal_products" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/{{ stream_slice.parent_id }}/products" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_stream" + parent_key: "id" + partition_field: "parent_id" + + +streams: + - "#/definitions/deals_stream" + - "#/definitions/deal_fields_stream" + - "#/definitions/files_stream" + - "#/definitions/filters_stream" + - "#/definitions/lead_labels_stream" + - "#/definitions/leads_stream" + - "#/definitions/notes_stream" + - "#/definitions/activities_stream" + - "#/definitions/activity_types_stream" + - "#/definitions/activity_fields_stream" + - "#/definitions/currencies_stream" + - "#/definitions/organization_stream" + - "#/definitions/organization_fields_stream" + - "#/definitions/permission_sets_stream" + - "#/definitions/persons_stream" + - "#/definitions/person_fields_stream" + - "#/definitions/pipelines_stream" + - "#/definitions/products_stream" + - "#/definitions/product_fields_stream" + - "#/definitions/roles_stream" + - "#/definitions/stages_stream" + - "#/definitions/users_stream" + - "#/definitions/deal_products_stream" + +check: + type: CheckStream + stream_names: + - "deals" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/pipedrive + connection_specification: + title: Pipedrive Spec + type: object + required: + - replication_start_date + - api_token + additionalProperties: true + properties: + api_token: + title: API Token + type: string + description: The Pipedrive API Token. + airbyte_secret: true + replication_start_date: + title: Start Date + description: UTC date and time in the format 2017-01-25T00:00:00Z. Any data + before this date will not be replicated. When specified and not None, then + stream will behave as incremental + examples: + - '2017-01-25 00:00:00Z' + type: string diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json index 6dd7a2100fa9..997124c2b50f 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json @@ -56,6 +56,18 @@ "sortable_flag": { "type": ["null", "boolean"] }, + "json_column_flag": { + "type": ["null", "boolean"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "id_suffix": { + "type": ["null", "string"] + }, + "is_subfield": { + "type": ["null", "boolean"] + }, "options": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json index f976f57b22f4..4b06133de601 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -14,16 +15,28 @@ "order_nr": { "type": ["null", "integer"] }, + "group_id": { + "type": ["null", "integer"] + }, "field_type": { "type": ["null", "string"] }, + "projects_detail_visible_flag": { + "type": ["null", "boolean"] + }, + "json_column_flag": { + "type": ["null", "boolean"] + }, "add_time": { + "format": "%Y-%m-%d %H:%M:%S", "type": ["null", "string"] }, "update_time": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "format": "%Y-%m-%d %H:%M:%S", + "type": ["null", "string"] + }, + "use_field": { + "type": ["null", "string"] }, "last_updated_by_user_id": { "type": ["null", "integer"] @@ -58,25 +71,38 @@ "sortable_flag": { "type": ["null", "boolean"] }, + "parent_id": { + "type": ["null", "integer"] + }, + "id_suffix": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "is_subfield": { + "type": ["null", "boolean"] + }, "options": { "type": ["null", "array"], "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string", "integer"] - }, - "label": { - "type": ["null", "string"] - } + "type": "object", + "additionalProperties": true + } + }, + "show_in_pipelines": { + "type": ["null", "object"], + "items": { + "show_in_all": { + "type": ["null", "boolean"] + }, + "pipeline_ids": { + "type": ["null", "array"] } } }, "mandatory_flag": { - "type": ["null", "string", "boolean", "object"] - }, - "pipeline_ids": { - "type": ["null", "array"] + "type": ["null", "object","boolean","integer"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json index 257c0b7c54c7..f24cb9d06a69 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json @@ -58,6 +58,21 @@ "sortable_flag": { "type": ["null", "boolean"] }, + "parent_id": { + "type": ["null", "integer"] + }, + "id_suffix": { + "type": ["null", "string"] + }, + "is_subfield": { + "type": ["null", "boolean"] + }, + "json_column_flag": { + "type": ["null", "boolean"] + }, + "group_id": { + "type": ["null", "integer"] + }, "options": { "type": ["null", "array"], "items": { @@ -77,6 +92,15 @@ }, "display_name": { "type": ["null", "string"] + }, + "use_field": { + "type": ["null", "string"] + }, + "display_field": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json index 5d6642fe0e3f..858901f328f1 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json @@ -132,9 +132,7 @@ "type": ["null", "string"] }, "update_time": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "type": ["null", "string"] }, "add_time": { "type": ["null", "string"] @@ -198,6 +196,12 @@ }, "cc_email": { "type": ["null", "string"] + }, + "category_id": { + "type": ["null", "integer"] + }, + "delete_time": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json index 7926e50d1019..811da2f9dcc9 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { "id": { @@ -17,26 +17,21 @@ "field_type": { "type": ["null", "string"] }, + "json_column_flag": { + "type": ["null", "boolean"] + }, "add_time": { "type": ["null", "string"] }, "update_time": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "type": ["null", "string"] }, "last_updated_by_user_id": { "type": ["null", "integer"] }, - "active_flag": { - "type": ["null", "boolean"] - }, "edit_flag": { "type": ["null", "boolean"] }, - "index_visible_flag": { - "type": ["null", "boolean"] - }, "details_visible_flag": { "type": ["null", "boolean"] }, @@ -49,31 +44,46 @@ "bulk_edit_allowed": { "type": ["null", "boolean"] }, - "searchable_flag": { - "type": ["null", "boolean"] - }, "filtering_allowed": { "type": ["null", "boolean"] }, "sortable_flag": { "type": ["null", "boolean"] }, + "mandatory_flag": { + "type": ["null", "boolean"] + }, + "link": { + "type": ["null", "string"] + }, + "use_field": { + "type": ["null", "string"] + }, + "display_field": { + "type": ["null", "string"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "index_visible_flag": { + "type": ["null", "boolean"] + }, + "searchable_flag": { + "type": ["null", "boolean"] + }, "options": { "type": ["null", "array"], "items": { - "type": ["null", "object"], + "type": "object", "properties": { "id": { - "type": ["null", "integer", "string", "boolean"] + "type": ["null", "boolean", "integer"] }, "label": { "type": ["null", "string"] } } } - }, - "mandatory_flag": { - "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json index 893aad9b7a56..43ea3ff38e9b 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json @@ -196,6 +196,12 @@ }, "cc_email": { "type": ["null", "string"] + }, + "picture_128_url": { + "type": ["null", "string"] + }, + "delete_time": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json index 811da2f9dcc9..add5c89700cf 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json @@ -77,7 +77,7 @@ "type": "object", "properties": { "id": { - "type": ["null", "boolean", "integer"] + "type": ["null", "string", "boolean", "integer"] }, "label": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json index 089c0cef654c..46fcaa14aab3 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json @@ -33,9 +33,7 @@ "type": ["null", "string"] }, "modified": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "type": ["null", "string"] }, "signup_flow_variation": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py index 2c9c76138d31..32dd3a077d5d 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py @@ -2,105 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from source_pipedrive.auth import QueryStringTokenAuthenticator -from source_pipedrive.streams import ( - Activities, - ActivityFields, - ActivityTypes, - Currencies, - DealFields, - DealProducts, - Deals, - Files, - Filters, - LeadLabels, - Leads, - Notes, - OrganizationFields, - Organizations, - PermissionSets, - PersonFields, - Persons, - Pipelines, - ProductFields, - Products, - Roles, - Stages, - Users, -) +WARNING: Do not modify this file. +""" -class SourcePipedrive(AbstractSource): - def _validate_and_transform(self, config: Mapping[str, Any]): - config["replication_start_date"] = pendulum.parse(config["replication_start_date"]) - return config - - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - config = self._validate_and_transform(config) - try: - stream = Deals(authenticator=self.get_authenticator(config), replication_start_date=config["replication_start_date"]) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records, None) - return True, None - except Exception as error: - return False, f"Unable to connect to Pipedrive API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - config = self._validate_and_transform(config) - stream_kwargs = {"authenticator": self.get_authenticator(config)} - incremental_kwargs = {**stream_kwargs, "replication_start_date": config["replication_start_date"]} - deals_stream = Deals(**incremental_kwargs) - streams = [ - Activities(**incremental_kwargs), - ActivityFields(**stream_kwargs), - ActivityTypes(**incremental_kwargs), - Currencies(**stream_kwargs), - deals_stream, - DealProducts(parent=deals_stream, **stream_kwargs), - DealFields(**stream_kwargs), - Files(**incremental_kwargs), - Filters(**incremental_kwargs), - LeadLabels(**stream_kwargs), - Leads(**stream_kwargs), - Notes(**incremental_kwargs), - Organizations(**incremental_kwargs), - OrganizationFields(**stream_kwargs), - PermissionSets(**stream_kwargs), - Persons(**incremental_kwargs), - PersonFields(**stream_kwargs), - Pipelines(**incremental_kwargs), - ProductFields(**stream_kwargs), - Products(**incremental_kwargs), - Roles(**stream_kwargs), - Stages(**incremental_kwargs), - Users(**incremental_kwargs), - ] - return streams - - @staticmethod - def get_authenticator(config: Mapping[str, Any]): - authorization = config.get("authorization") - if authorization: - if authorization["auth_type"] == "Client": - return Oauth2Authenticator( - token_refresh_endpoint="https://oauth.pipedrive.com/oauth/token", - client_id=authorization["client_id"], - client_secret=authorization["client_secret"], - refresh_token=authorization["refresh_token"], - ) - elif authorization["auth_type"] == "Token": - return QueryStringTokenAuthenticator(api_token=authorization["api_token"]) - # backward compatibility - return QueryStringTokenAuthenticator(api_token=config["api_token"]) +# Declarative Source +class SourcePipedrive(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json deleted file mode 100644 index be9a09b98af1..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/pipedrive", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pipedrive Spec", - "type": "object", - "required": ["replication_start_date"], - "additionalProperties": true, - "properties": { - "authorization": { - "type": "object", - "title": "API Key Authentication", - "required": ["auth_type", "api_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "Token", - "order": 0 - }, - "api_token": { - "title": "API Token", - "type": "string", - "description": "The Pipedrive API Token.", - "airbyte_secret": true - } - } - }, - "replication_start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - } - } - }, - "supportsIncremental": true -} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py deleted file mode 100755 index bc505e8c2b56..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py +++ /dev/null @@ -1,253 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - -PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/" - - -class PipedriveStream(HttpStream, ABC): - url_base = PIPEDRIVE_URL_BASE - primary_key = "id" - data_field = "data" - page_size = 50 - - def __init__(self, replication_start_date=None, **kwargs): - super().__init__(**kwargs) - self._replication_start_date = replication_start_date - - @property - def cursor_field(self) -> Union[str, List[str]]: - if self._replication_start_date: - return "update_time" - return [] - - def path(self, **kwargs) -> str: - if self._replication_start_date: - return "recents" - - class_name = self.__class__.__name__ - return f"{class_name[0].lower()}{class_name[1:]}" - - @property - def path_param(self): - return self.name[:-1] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query - the next page in the response. - If there are no more pages in the result, return None. - """ - pagination_data = response.json().get("additional_data", {}).get("pagination", {}) - if pagination_data.get("more_items_in_collection") and pagination_data.get("start") is not None: - start = pagination_data.get("start") + self.page_size - return {"start": start} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - params = {"limit": self.page_size, **next_page_token} - - replication_start_date = self._replication_start_date - if replication_start_date: - cursor_value = stream_state.get(self.cursor_field) - if cursor_value: - cursor_value = pendulum.parse(cursor_value) - replication_start_date = max(replication_start_date, cursor_value) - - params.update( - { - "items": self.path_param, - "since_timestamp": replication_start_date.strftime("%Y-%m-%d %H:%M:%S"), - } - ) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ - records = response.json().get(self.data_field) or [] - for record in records: - record = record.get(self.data_field) or record - if self.primary_key in record and record[self.primary_key] is None: - # Convert "id: null" fields to "id: 0" since id is primary key and SAT checks if it is not null. - record[self.primary_key] = 0 - yield record - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - replication_start_date = self._replication_start_date.strftime("%Y-%m-%d %H:%M:%S") - current_stream_state[self.cursor_field] = max( - latest_record.get(self.cursor_field, replication_start_date), - current_stream_state.get(self.cursor_field, replication_start_date), - ) - return current_stream_state - - -class Deals(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Deals#getDeals, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class DealFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/DealFields#getDealFields""" - - -class Files(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Files#getFiles - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Filters(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Filters#getFilters - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class LeadLabels(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/LeadLabels#getLeadLabels""" - - -class Leads(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Leads#getLeads""" - - -class Notes(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Notes#getNotes - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Activities(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Activities#getActivities, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - path_param = "activity" - - -class ActivityFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields""" - - -class ActivityTypes(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/ActivityTypes#getActivityTypes - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - path_param = "activityType" - - -class Currencies(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Currencies#getCurrencies""" - - -class Organizations(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Organizations#getOrganizations, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class OrganizationFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/OrganizationFields#getOrganizationFields""" - - -class PermissionSets(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/PermissionSets#getPermissionSets""" - - -class Persons(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Persons#getPersons, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class PersonFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/PersonFields#getPersonFields""" - - -class Pipelines(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Pipelines#getPipelines, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Products(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Products#getProducts, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class ProductFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/ProductFields#getProductFields""" - - -class Roles(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Roles#getRoles""" - - -class Stages(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Stages#getStages, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Users(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Users#getUsers, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - cursor_field = "modified" - page_size = 500 - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - record_gen = super().parse_response(response=response, **kwargs) - for records in record_gen: - yield from records - - -class DealProducts(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Deals#getDealProducts""" - - def __init__(self, parent, **kwargs): - self.parent = parent - super().__init__(**kwargs) - - def path(self, stream_slice, **kwargs) -> str: - return f"deals/{stream_slice['deal_id']}/products" - - def stream_slices(self, sync_mode, cursor_field=None, stream_state=None): - stream_slices = self.parent.stream_slices(sync_mode=SyncMode.full_refresh) - for stream_slice in stream_slices: - records = self.parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) - for record in records: - if record["products_count"]: - yield {"deal_id": record["id"]} diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/conftest.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/conftest.py deleted file mode 100644 index c11da1ff0380..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/unit_tests/conftest.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pendulum -import pytest -from source_pipedrive.source import SourcePipedrive - -replication_start_date = "2017-01-25T00:00:00Z" - - -@pytest.fixture -def config_oauth(): - return { - "authorization": { - "auth_type": "Client", - "client_id": "6779ef20e75817b79602", - "client_secret": "7607999ef26581e81726777b7b79f20e70e75602", - "refresh_token": "refresh_token", - }, - "replication_start_date": replication_start_date, - } - - -@pytest.fixture -def config_token(): - return { - "authorization": { - "auth_type": "Token", - "api_token": "api_token" - }, - "replication_start_date": replication_start_date - } - - -@pytest.fixture -def stream_kwargs(config_token): - return {"authenticator": SourcePipedrive.get_authenticator(config_token)} - - -@pytest.fixture -def incremental_kwargs(config_token): - return { - "authenticator": SourcePipedrive.get_authenticator(config_token), - "replication_start_date": pendulum.parse(config_token["replication_start_date"]) - } diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_source.py deleted file mode 100644 index f65cf6cfdbb8..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_source.py +++ /dev/null @@ -1,72 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - ConnectorSpecification, - DestinationSyncMode, - SyncMode, -) -from source_pipedrive.source import SourcePipedrive - -logger = AirbyteLogger() - -PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/" - - -def test_check_connection(requests_mock, config_token): - body = {"success": "true", "data": [{"id": 1, "update_time": "2020-10-14T11:30:36.551Z"}]} - response = setup_response(200, body) - api_token = config_token["authorization"]["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + "recents?limit=50&api_token=" + api_token, response) - - ok, error = SourcePipedrive().check_connection(logger, config_token) - - assert ok - assert not error - - -def test_check_connection_exception(requests_mock, config_token): - response = setup_response(400, {}) - api_token = config_token["authorization"]["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + "recents?limit=50&api_token=" + api_token, response) - - ok, error = SourcePipedrive().check_connection(logger, config_token) - - assert not ok - assert error - - -def test_streams(config_token): - streams = SourcePipedrive().streams(config_token) - - assert len(streams) == 23 - - -def setup_response(status, body): - return [ - {"json": body, "status_code": status}, - ] - - -def test_spec(): - spec = SourcePipedrive().spec(logger) - assert isinstance(spec, ConnectorSpecification) - - -def test_read(config_token): - source = SourcePipedrive() - catalog = ConfiguredAirbyteCatalog( - streams=[ - ConfiguredAirbyteStream( - stream=AirbyteStream(name="deals", json_schema={}, supported_sync_modes=["full_refresh", "incremental"]), - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.overwrite, - ) - ] - ) - assert source.read(logger, config_token, catalog) diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_streams.py deleted file mode 100644 index 53796150dc87..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_streams.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from airbyte_cdk.models import SyncMode -from source_pipedrive.streams import ( - Activities, - ActivityFields, - DealFields, - Deals, - Leads, - OrganizationFields, - Organizations, - PersonFields, - Persons, - PipedriveStream, - Pipelines, - Stages, -) - -PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/" - - -def test_cursor_field_incremental(incremental_kwargs): - stream = PipedriveStream(**incremental_kwargs) - - assert stream.cursor_field == "update_time" - - -def test_cursor_field_refresh(stream_kwargs): - stream = PipedriveStream(**stream_kwargs) - - assert stream.cursor_field == [] - - -def test_path_incremental(incremental_kwargs): - stream = PipedriveStream(**incremental_kwargs) - - assert stream.path() == "recents" - - -def test_path_refresh(stream_kwargs): - stream = PipedriveStream(**stream_kwargs) - - assert stream.path() == "pipedriveStream" - - -@pytest.mark.parametrize( - "stream, endpoint", - [ - (ActivityFields, "activityFields"), - (DealFields, "dealFields"), - (OrganizationFields, "organizationFields"), - (PersonFields, "personFields"), - (Leads, "leads"), - ], -) -def test_streams_full_refresh(stream, endpoint, requests_mock, stream_kwargs): - body = { - "success": "true", - "data": [{"id": 1, "update_time": "2020-10-14T11:30:36.551Z"}, {"id": 2, "update_time": "2020-10-14T11:30:36.551Z"}], - } - - response = setup_response(200, body) - - api_token = stream_kwargs["authenticator"].params["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + endpoint + "?limit=50&api_token=" + api_token, response) - - stream = stream(**stream_kwargs) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - - assert records - - -@pytest.mark.parametrize( - "stream", - [ - Activities, - Deals, - Organizations, - Persons, - Pipelines, - Stages, - # Users - ], -) -def test_streams_incremental_sync(stream, requests_mock, incremental_kwargs): - body = { - "success": "true", - "data": [{"id": 1, "update_time": "2020-10-14T11:30:36.551Z"}, {"id": 2, "update_time": "2020-11-14T11:30:36.551Z"}], - } - - response = setup_response(200, body) - - api_token = incremental_kwargs["authenticator"].params["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + "recents?limit=50&api_token=" + api_token, response) - - stream = stream(**incremental_kwargs) - records = stream.read_records(sync_mode=SyncMode.incremental) - stream_state = {} - for record in records: - stream_state = stream.get_updated_state(stream_state, latest_record=record) - - assert records - assert stream_state["update_time"] == "2020-11-14T11:30:36.551Z" - - -def setup_response(status, body): - return [ - {"json": body, "status_code": status}, - ] diff --git a/docs/integrations/sources/pipedrive-migrations.md b/docs/integrations/sources/pipedrive-migrations.md new file mode 100644 index 000000000000..d4cef6e3242d --- /dev/null +++ b/docs/integrations/sources/pipedrive-migrations.md @@ -0,0 +1,6 @@ +# Pipedrive Migration Guide + +## Upgrading to 2.0.0 +Please update your config and reset your data (to match the new format). This version has changed the config to only require an API key. + +This version also removes the `pipeline_ids` field from the `deal_fields` stream. diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md index 534f992294fb..48ce6860a4cd 100644 --- a/docs/integrations/sources/pipedrive.md +++ b/docs/integrations/sources/pipedrive.md @@ -112,6 +112,7 @@ The Pipedrive connector will gracefully handle rate limits. For more information | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 2.0.0 | 2023-08-09 | [29293](https://github.com/airbytehq/airbyte/pull/29293) | Migrated to Low-Code CDK | | 1.0.0 | 2023-06-29 | [27832](https://github.com/airbytehq/airbyte/pull/27832) | Remove `followers_count` field from `Products` stream | | 0.1.19 | 2023-07-05 | [27967](https://github.com/airbytehq/airbyte/pull/27967) | Update `OrganizationFields` and `ProductFields` with `display_field` field | | 0.1.18 | 2023-06-02 | [26892](https://github.com/airbytehq/airbyte/pull/26892) | Update `DialFields` schema with `pipeline_ids` property |