Skip to content

Commit

Permalink
Source Sendgrid: Update authenticator (#38478)
Browse files Browse the repository at this point in the history
Co-authored-by: Oleksandr Bazarnov <[email protected]>
  • Loading branch information
ChristoGrab and bazarnov authored May 22, 2024
1 parent ce08b24 commit d8db2c2
Show file tree
Hide file tree
Showing 7 changed files with 349 additions and 52 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87
dockerImageTag: 1.0.1
dockerImageTag: 1.0.2
releases:
breakingChanges:
1.0.0:
Expand Down
356 changes: 321 additions & 35 deletions airbyte-integrations/connectors/source-sendgrid/poetry.lock

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
build-backend = "poetry.core.masonry.api"

[tool.poetry]
version = "1.0.1"
version = "1.0.2"
name = "source-sendgrid"
description = "Source implementation for Sendgrid."
authors = [ "Airbyte <[email protected]>",]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
from airbyte_cdk.sources.streams import Stream
from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator

from .streams import Contacts

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def path(self, **kwargs) -> str:

@default_backoff_handler(max_tries=5, factor=15)
def _send_http_request(self, method: str, url: str, stream: bool = False, enable_auth: bool = True):
headers = self.authenticator.get_auth_header() if enable_auth else None
headers = self._session.auth.get_auth_header() if enable_auth else None
response = self._session.request(method, url=url, headers=headers, stream=stream)
if response.status_code not in [200, 202]:
self.logger.error(f"error body: {response.text}")
Expand Down Expand Up @@ -159,9 +159,8 @@ def download_data(self, url: str, chunk_size: int = 1024) -> tuple[str, str]:

url_parsed = urlparse(url)
tmp_file = os.path.realpath(os.path.basename(url_parsed.path[1:-5]))
with closing(self._send_http_request("GET", f"{url}", stream=True, enable_auth=False)) as response, open(
tmp_file, "wb"
) as data_file:
download_session = requests.get(f"{url}", stream=True)
with closing(download_session) as response, open(tmp_file, "wb") as data_file:
for chunk in response.iter_content(chunk_size=chunk_size):
try:
# see if it's compressed. we are seeing some that are not all of a sudden.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,14 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#

import json
import logging
import os
import unittest
from unittest.mock import MagicMock, Mock, patch
from unittest.mock import MagicMock, Mock

import pandas as pd
import pendulum
import pytest
import requests
from airbyte_cdk.models import SyncMode
from numpy import nan
from requests import codes
Expand Down Expand Up @@ -47,6 +45,13 @@ def mock_pendulum_now(monkeypatch):
monkeypatch.setattr(pendulum, "now", pendulum_mock)


@pytest.fixture()
def mock_authenticator():
mock = Mock()
mock.get_auth_header.return_value = {"Authorization": "Bearer fake_token"}
return mock


def test_source_wrong_credentials():
source = SourceSendgrid()
status, error = source.check_connection(logger=logging.getLogger("airbyte"), config={"api_key": "wrong.api.key123"})
Expand Down Expand Up @@ -76,8 +81,9 @@ def test_read_records(
stream_name,
url,
expected,
requests_mock,
requests_mock
):
requests_mock.get("https://api.sendgrid.com/v3/marketing/contacts/exports", json={})
stream = find_stream(stream_name)
requests_mock.get("https://api.sendgrid.com/v3/marketing", json={})
requests_mock.get(url, json={"name": "test", "id": "id", "contact_count": 20, "_metadata": {"self": "self"}})
Expand Down Expand Up @@ -131,8 +137,10 @@ def test_should_retry_on_permission_error(stream_name, status, expected):
assert stream.retriever.requester._should_retry(response_mock) == expected


def test_compressed_contact_response(requests_mock):
def test_compressed_contact_response(requests_mock, mock_authenticator):
stream = Contacts()
stream._session.auth = mock_authenticator

with open(os.path.dirname(__file__) + "/compressed_response", "rb") as file_response:
url = "https://api.sendgrid.com/v3/marketing/contacts/exports"
requests_mock.register_uri("POST", url, [{"json": {"id": "random_id"}, "status_code": 202}])
Expand All @@ -152,8 +160,10 @@ def test_compressed_contact_response(requests_mock):
assert recs == expected_records


def test_uncompressed_contact_response(requests_mock):
def test_uncompressed_contact_response(requests_mock, mock_authenticator):
stream = Contacts()
stream._session.auth = mock_authenticator

with open(os.path.dirname(__file__) + "/decompressed_response.csv", "rb") as file_response:
url = "https://api.sendgrid.com/v3/marketing/contacts/exports"
requests_mock.register_uri("POST", url, [{"json": {"id": "random_id"}, "status_code": 202}])
Expand All @@ -173,8 +183,9 @@ def test_uncompressed_contact_response(requests_mock):
assert recs == expected_records


def test_bad_job_response(requests_mock):
def test_bad_job_response(requests_mock, mock_authenticator):
stream = Contacts()
stream._session.auth = mock_authenticator
url = "https://api.sendgrid.com/v3/marketing/contacts/exports"

requests_mock.register_uri(
Expand Down
7 changes: 4 additions & 3 deletions docs/integrations/sources/sendgrid.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,10 @@ The connector is restricted by normal Sendgrid [requests limitation](https://doc

| Version | Date | Pull Request | Subject |
| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| 1.0.1 | 2024-05-20 | [38264](https://github.com/airbytehq/airbyte/pull/38264) | Replace AirbyteLogger with logging.Logger |
| 1.0.0 | 2024-04-15 | [35776](https://github.com/airbytehq/airbyte/pull/35776) | Migration to low-code CDK. Breaking change that updates configuration keys, removes unsubscribe_groups stream, renames a stream to singlesend_stats, and adds the singlesends stream. |
| 0.5.0 | 2024-03-26 | [36455](https://github.com/airbytehq/airbyte/pull/36455) | Unpin CDK version, add record counts to state messages |
| 1.0.2 | 2024-05-21 | [38478](https://github.com/airbytehq/airbyte/pull/38478) | Update deprecated authenticator package |
| 1.0.1 | 2024-05-20 | [38264](https://github.com/airbytehq/airbyte/pull/38264) | Replace AirbyteLogger with logging.Logger |
| 1.0.0 | 2024-04-15 | [35776](https://github.com/airbytehq/airbyte/pull/35776) | Migration to low-code CDK. Breaking change that updates configuration keys, removes unsubscribe_groups stream, renames a stream to singlesend_stats, and adds the singlesends stream. |
| 0.5.0 | 2024-03-26 | [36455](https://github.com/airbytehq/airbyte/pull/36455) | Unpin CDK version, add record counts to state messages |
| 0.4.3 | 2024-02-21 | [35181](https://github.com/airbytehq/airbyte/pull/35343) | Handle uncompressed contacts downloads. |
| 0.4.2 | 2024-02-12 | [35181](https://github.com/airbytehq/airbyte/pull/35181) | Manage dependencies with Poetry. |
| 0.4.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image |
Expand Down

0 comments on commit d8db2c2

Please sign in to comment.