From 8eee808dfaa4ecd23fbc26e32480bb8c16d791af Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Mon, 19 Feb 2024 14:28:44 +0200 Subject: [PATCH 1/7] added fields transformer for type fields in users and blocks streams --- .../schemas/shared/rich_text.json | 11 +++++++ .../source_notion/schemas/shared/user.json | 28 ++++++++++++++++ .../source-notion/source_notion/streams.py | 33 ++++++++++++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json index cc003048b875..e8ecf832da80 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json @@ -32,6 +32,17 @@ "properties": { "type": { "type": ["null", "string"] + }, + "info": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + } + } } } }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json index d893b1147b7e..8c5000dba205 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json @@ -37,10 +37,38 @@ "type": { "type": "string" }, + "info": { + "avatar_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "person": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, "workspace": { "type": ["null", "boolean"] } } + }, + "workspace_name": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index a0546d2116c4..f3ec40c2ce56 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -252,6 +252,23 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> params["start_cursor"] = next_page_token["next_cursor"] return params + def transform(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + bot = record.get("bot") + if bot: + owner = bot.get("owner", {}) + owner_type = owner.get("type") + if owner_type: + owner_info = owner[owner_type] + record["bot"]["owner"]["info"] = owner_info + del record["bot"]["owner"][owner_type] + return record + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + # sometimes notion api returns response without results object + data = response.json().get("results", []) + for record in data: + yield self.transform(record) + class Databases(IncrementalNotionStream): """ @@ -313,6 +330,20 @@ def stream_slices( yield {"page_id": page_id} + def transform(self, record: Mapping[str, Any]): + transform_object_field = record["type"] + rich_text = record.get(transform_object_field, {}).get("rich_text", {}) + + if rich_text: + for r in rich_text: + mention = r.get("mention") + if mention: + type_info = mention[mention["type"]] + record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"]["info"] = type_info + del record[transform_object_field]["rich_text"][rich_text.index(r)]["mention"][mention["type"]] + + return record + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: # pages and databases blocks are already fetched in their streams, so no # need to do it again @@ -321,7 +352,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, records = super().parse_response(response, stream_state=stream_state, **kwargs) for record in records: if record["type"] not in ("child_page", "child_database", "ai_block"): - yield record + yield self.transform(record) def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: # if reached recursive limit, don't read anymore From 2ca8456c966ee1ab264a33aea150707617d88312 Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Mon, 19 Feb 2024 14:29:10 +0200 Subject: [PATCH 2/7] added unit tests for new methods --- .../source-notion/unit_tests/test_streams.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index ceeb5b1ca5cb..d369b201becc 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -312,3 +312,41 @@ def test_request_throttle(initial_page_size, expected_page_size, mock_response, stream.should_retry(response=response) assert stream.page_size == expected_page_size + + +def test_users_record_transformer(): + stream = Users(config=MagicMock()) + response_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "user": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + expected_record = { + "object": "user", "id": "id", "name": "Airbyte", "avatar_url": "some url", "type": "bot", + "bot": {"owner": {"type": "user", "info": {"object": "user", "id": "id", "name": "Test User", "avatar_url": None, "type": "person", + "person": {"email": "email"}}}, "workspace_name": "test"} + } + assert stream.transform(response_record) == expected_record + + +def test_block_record_transformer(): + stream = Blocks(parent=None, config=MagicMock()) + response_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "page": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, + "plain_text": "test", "href": "https://www.notion.so/id"}], "color": "default"} + } + expected_record = { + "object": "block", "id": "id", "parent": {"type": "page_id", "page_id": "id"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", + "created_by": {"object": "user", "id": "id"}, "last_edited_by": {"object": "user", "id": "id"}, "has_children": False, "archived": False, "type": "paragraph", + "paragraph": {"rich_text": [{"type": "text", "text": {"content": "test", "link": None}, "annotations":{"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text":"test", "href": None}, + {"type": "text", "text": {"content": "@", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": True, "color": "default"}, "plain_text": "@", "href": None}, + {"type": "text", "text": {"content": "test", "link": None}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": None}, + {"type": "mention", "mention": {"type": "page", "info": {"id": "id"}}, "annotations": {"bold": False, "italic": False, "strikethrough": False, "underline": False, "code": False, "color": "default"}, "plain_text": "test", "href": "https://www.notion.so/id"}], + "color": "default"} + } + assert stream.transform(response_record) == expected_record From 571377c170c41175132d5fead032bae27721ea22 Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Mon, 19 Feb 2024 14:29:37 +0200 Subject: [PATCH 3/7] updated expected records --- .../source-notion/integration_tests/expected_records.jsonl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl index 27bf2dc60ca8..9989f84792de 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl @@ -1,9 +1,9 @@ {"stream": "users", "data": {"object": "user", "id": "5612c094-99ec-4ba3-ac7f-df8d84c8d6be", "name": "Sherif Nada", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png", "type": "person", "person": {"email": "sherif@airbyte.io"}}, "emitted_at": 1697023279924} {"stream": "users", "data": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}, "emitted_at": 1697023279925} {"stream": "users", "data": {"object": "user", "id": "c1ff0160-b2af-497a-aab7-8b61e625e4e3", "name": "Gil Cho", "avatar_url": "https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100", "type": "person", "person": {"email": "gil@airbyte.io"}}, "emitted_at": 1697023279925} -{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1697023281967} -{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green"}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow"}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red"}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default"}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default"}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default"}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default"}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default"}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1697023281968} -{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red"}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue"}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1697023281968} +{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green", "description": null}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow", "description": null}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red", "description": null}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default", "description": null}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default", "description": null}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default", "description": null}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default", "description": null}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default", "description": null}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1708341487319} +{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red", "description": null}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green", "description": null}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue", "description": null}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow", "description": null}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange", "description": null}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1708341487319} {"stream": "pages", "data": {"object": "page", "id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Daily Standup", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Daily Standup", "href": null}]}}], "url": "https://www.notion.so/Daily-Standup-39a69b4e7cc24f7aa656dd128f3ce855", "public_url": null}, "emitted_at": 1697023284463} {"stream": "pages", "data": {"object": "page", "id": "621d3dc4-55fe-46ce-a3ff-83da06e5f9fb", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Sprint Planning ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Sprint Planning ", "href": null}]}}], "url": "https://www.notion.so/Sprint-Planning-621d3dc455fe46cea3ff83da06e5f9fb", "public_url": null}, "emitted_at": 1697023284465} {"stream": "pages", "data": {"object": "page", "id": "6eb2dedc-8b88-486c-8648-d1878bafb106", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Ad Hoc Meeting", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ad Hoc Meeting", "href": null}]}}], "url": "https://www.notion.so/Ad-Hoc-Meeting-6eb2dedc8b88486c8648d1878bafb106", "public_url": null}, "emitted_at": 1697023284465} From 2502531a634c93abe486a207cd668379d536ffbc Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Mon, 19 Feb 2024 14:29:55 +0200 Subject: [PATCH 4/7] updated notion version --- .../connectors/source-notion/metadata.yaml | 2 +- docs/integrations/sources/notion.md | 81 ++++++++++--------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index 8a95d375f3cf..58b329ffe793 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 2.0.9 + dockerImageTag: 2.1.0 dockerRepository: airbyte/source-notion documentationUrl: https://docs.airbyte.com/integrations/sources/notion githubIssueLabel: source-notion diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index 84319c018bc9..e4b6154d7497 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -110,43 +110,44 @@ The connector is restricted by Notion [request limits](https://developers.notion ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------- | -| 2.0.9 | 2024-02-12 | [35155](https://github.com/airbytehq/airbyte/pull/35155) | Manage dependencies with Poetry. | -| 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | -| 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | -| 2.0.6 | 2023-10-25 | [31825](https://github.com/airbytehq/airbyte/pull/31825) | Increase max_retries on retryable errors | -| 2.0.5 | 2023-10-23 | [31742](https://github.com/airbytehq/airbyte/pull/31742) | Add 'synced_block' property to Blocks schema | -| 2.0.4 | 2023-10-19 | [31625](https://github.com/airbytehq/airbyte/pull/31625) | Fix check_connection method | -| 2.0.3 | 2023-10-19 | [31612](https://github.com/airbytehq/airbyte/pull/31612) | Add exponential backoff for 500 errors | -| 2.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 2.0.1 | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks | -| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | -| 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | -| 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | -| 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy | -| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | -| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | -| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | -| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | -| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | -| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | -| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | -| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | -| 1.0.5 | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream | -| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | -| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | -| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | -| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | -| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | -| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | -| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | -| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | -| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | -| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | -| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | -| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | -| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | -| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| 2.1.0 | 2024-02-19 | | Update users stream schema with bot type info fields and block schema with mention type info fields. | +| 2.0.9 | 2024-02-12 | [35155](https://github.com/airbytehq/airbyte/pull/35155) | Manage dependencies with Poetry. | +| 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | +| 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | +| 2.0.6 | 2023-10-25 | [31825](https://github.com/airbytehq/airbyte/pull/31825) | Increase max_retries on retryable errors | +| 2.0.5 | 2023-10-23 | [31742](https://github.com/airbytehq/airbyte/pull/31742) | Add 'synced_block' property to Blocks schema | +| 2.0.4 | 2023-10-19 | [31625](https://github.com/airbytehq/airbyte/pull/31625) | Fix check_connection method | +| 2.0.3 | 2023-10-19 | [31612](https://github.com/airbytehq/airbyte/pull/31612) | Add exponential backoff for 500 errors | +| 2.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.0.1 | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks | +| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | +| 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | +| 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | +| 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy | +| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | +| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | +| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | +| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | +| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | +| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | +| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | +| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | +| 1.0.5 | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream | +| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | +| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | +| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | +| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | +| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | +| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | +| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | +| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | +| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | +| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | +| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | +| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | +| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | +| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | From 1660a27a07bab6319ae251900fc58ab69214a312 Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Mon, 19 Feb 2024 14:37:02 +0200 Subject: [PATCH 5/7] updated changelog --- docs/integrations/sources/notion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index e4b6154d7497..04d96e2a9be2 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -112,7 +112,7 @@ The connector is restricted by Notion [request limits](https://developers.notion | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| -| 2.1.0 | 2024-02-19 | | Update users stream schema with bot type info fields and block schema with mention type info fields. | +| 2.1.0 | 2024-02-19 | [35409](https://github.com/airbytehq/airbyte/pull/35409) | Update users stream schema with bot type info fields and block schema with mention type info fields. | | 2.0.9 | 2024-02-12 | [35155](https://github.com/airbytehq/airbyte/pull/35155) | Manage dependencies with Poetry. | | 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | | 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | From 2c70306183908dd726bd8e5627374be225edb772 Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Mon, 19 Feb 2024 20:02:15 +0200 Subject: [PATCH 6/7] updated transform methods --- .../source-notion/source_notion/streams.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index f3ec40c2ce56..f7cb9e456e25 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -253,12 +253,11 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> return params def transform(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: - bot = record.get("bot") - if bot: - owner = bot.get("owner", {}) + owner = record.get("bot", {}).get("owner") + if owner: owner_type = owner.get("type") - if owner_type: - owner_info = owner[owner_type] + owner_info = owner.get(owner_type) + if owner_type and owner_info: record["bot"]["owner"]["info"] = owner_info del record["bot"]["owner"][owner_type] return record @@ -330,11 +329,11 @@ def stream_slices( yield {"page_id": page_id} - def transform(self, record: Mapping[str, Any]): - transform_object_field = record["type"] - rich_text = record.get(transform_object_field, {}).get("rich_text", {}) + def transform(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + transform_object_field = record.get("type") - if rich_text: + if transform_object_field: + rich_text = record.get(transform_object_field, {}).get("rich_text", []) for r in rich_text: mention = r.get("mention") if mention: From cedabc2d13f13e42b956fd957c69111e6a6e71f5 Mon Sep 17 00:00:00 2001 From: darynaishchenko Date: Wed, 21 Feb 2024 15:04:33 +0200 Subject: [PATCH 7/7] bump version in pyproject.toml --- airbyte-integrations/connectors/source-notion/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-notion/pyproject.toml b/airbyte-integrations/connectors/source-notion/pyproject.toml index 4218b07b35ba..3c4250763099 100644 --- a/airbyte-integrations/connectors/source-notion/pyproject.toml +++ b/airbyte-integrations/connectors/source-notion/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "2.0.9" +version = "2.1.0" name = "source-notion" description = "Source implementation for Notion." authors = [ "Airbyte ",]