diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile index 278e6f66b309..2600a9cf695d 100644 --- a/airbyte-integrations/connectors/source-notion/Dockerfile +++ b/airbyte-integrations/connectors/source-notion/Dockerfile @@ -34,5 +34,5 @@ COPY source_notion ./source_notion ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.3.0 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-notion diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml index 332e93b9e646..7853ba8c8d64 100644 --- a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml @@ -17,12 +17,14 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 1.3.0 basic_read: tests: - config_path: "secrets/config.json" expect_records: path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false + fail_on_extra_columns: true incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index 2a421ede2c1b..768f819081bb 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 1.3.0 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-notion githubIssueLabel: source-notion icon: notion.svg @@ -17,6 +17,11 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 2.0.0: + message: "Version 2.0.0 introduces schema changes to multiple properties shared by the blocks, databases and pages streams. These changes were introduced to reflect updates to the Notion API. A full schema refresh and data reset are required to upgrade to this version." + upgradeDeadline: "2023-11-09" suggestedStreams: streams: - blocks diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json index 7da7ba939c59..177adee4fdd3 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -13,10 +13,18 @@ "type": "string" }, "created_time": { - "type": "string" + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" + }, + "last_edited_by": { + "$ref": "user.json" }, "last_edited_time": { - "type": "string" + "type": "string", + "format": "date-time" }, "archived": { "type": "boolean" @@ -60,36 +68,32 @@ "video" ] }, - "paragraph": { "$ref": "text_element.json" }, - "quote": { "$ref": "text_element.json" }, - "bulleted_list_item": { "$ref": "text_element.json" }, - "numbered_list_item": { "$ref": "text_element.json" }, - "toggle": { "$ref": "text_element.json" }, - "heading_1": { "$ref": "heading.json" }, - "heading_2": { "$ref": "heading.json" }, - "heading_3": { "$ref": "heading.json" }, - "callout": { + "bookmark": { "type": "object", "properties": { - "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, - "icon": { "$ref": "icon.json" }, - "children": { "type": "array", "items": { "type": "object" } } + "url": { "type": "string" }, + "caption": { "type": "array", "items": { "$ref": "rich_text.json" } } } }, - "to_do": { + "breadcrumb": { + "type": "object" + }, + "bulleted_list_item": { "$ref": "text_element.json" }, + "callout": { "type": "object", "properties": { - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, - "checked": { "type": ["null", "boolean"] }, - "children": { "type": "array", "items": { "type": "object" } } + "color": { "type": "string" }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "icon": { "$ref": "icon.json" } } }, + "child_page": { "$ref": "child.json" }, + "child_database": { "$ref": "child.json" }, "code": { "type": "object", "properties": { - "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "caption": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "language": { "enum": [ "abap", @@ -168,30 +172,88 @@ } } }, - "child_page": { "$ref": "child.json" }, - "child_database": { "$ref": "child.json" }, + "column": { + "type": "object" + }, + "column_list": { + "type": "object" + }, + "divider": { + "type": "object" + }, "embed": { "type": "object", "properties": { "url": { "type": "string" } } }, - "image": { "$ref": "file.json" }, - "video": { "$ref": "file.json" }, + "equation": { + "type": "object", + "properties": { + "expression": { "type": "string" } + } + }, "file": { "$ref": "file.json" }, + "heading_1": { "$ref": "heading.json" }, + "heading_2": { "$ref": "heading.json" }, + "heading_3": { "$ref": "heading.json" }, + "image": { "$ref": "file.json" }, + "link_preview": { + "type": "object", + "properties": { + "url": { "type": "string" } + } + }, + "link_to_page": { + "type": "object", + "properties": { + "page_id": { "type": "string" }, + "type": { "type": "string" } + } + }, + "numbered_list_item": { "$ref": "text_element.json" }, + "paragraph": { "$ref": "text_element.json" }, "pdf": { "$ref": "file.json" }, - "bookmark": { + "quote": { "$ref": "text_element.json" }, + "table": { "type": "object", "properties": { - "url": { "type": "string" }, - "caption": { "type": "array", "items": { "$ref": "rich_text.json" } } + "table_width": { "type": "integer" }, + "has_column_header": { "type": "boolean" }, + "has_row_header": { "type": "boolean" } } }, - "equation": { + "table_of_contents": { "type": "object", "properties": { - "expression": { "type": "string" } + "color": { "type": "string" } + } + }, + "table_row": { + "type": "object", + "properties": { + "cells": { "type": "array", "items": { "$ref": "rich_text.json" } } + } + }, + "template": { + "type": "object", + "properties": { + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } } + } + }, + "to_do": { + "type": "object", + "properties": { + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "checked": { "type": ["null", "boolean"] }, + "color": { "type": "string" }, + "children": { "type": "array", "items": { "type": "object" } } } + }, + "toggle": { "$ref": "text_element.json" }, + "video": { "$ref": "file.json" }, + "unsupported": { + "type": "object" } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json index 8b521d46fcb9..55f004c52241 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json @@ -10,10 +10,12 @@ "type": "string" }, "created_time": { - "type": "string" + "type": "string", + "format": "date-time" }, "last_edited_time": { - "type": "string" + "type": "string", + "format": "date-time" }, "title": { "type": "array", @@ -21,6 +23,12 @@ "$ref": "rich_text.json" } }, + "description": { + "type": "array", + "items": { + "$ref": "rich_text.json" + } + }, "last_edited_by": { "$ref": "user.json" }, @@ -45,6 +53,9 @@ "is_inline": { "type": ["null", "boolean"] }, + "public_url": { + "type": ["null", "string"] + }, "properties": { "type": "array", "items": { @@ -57,7 +68,7 @@ "value": { "type": "object", "additionalProperties": true, - "oneOf": [ + "anyOf": [ { "type": "object", "additionalProperties": true, @@ -67,19 +78,26 @@ }, "type": { "enum": [ - "title", - "rich_text", - "date", - "people", - "files", "checkbox", - "url", - "email", - "phone_number", - "created_time", "created_by", + "created_time", + "date", + "email", + "files", + "formula", + "last_edited_by", "last_edited_time", - "last_edited_by" + "multi_select", + "number", + "people", + "phone_number", + "relation", + "rich_text", + "rollup", + "select", + "status", + "title", + "url" ] }, "name": { @@ -87,6 +105,159 @@ } } }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["checkbox"] + }, + "name": { + "type": "string" + }, + "checkbox": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["created_by"] + }, + "name": { + "type": "string" + }, + "created_by": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["created_time"] + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["date"] + }, + "name": { + "type": "string" + }, + "date": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["email"] + }, + "name": { + "type": "string" + }, + "email": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["files"] + }, + "name": { + "type": "string" + }, + "files": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["formula"] + }, + "name": { + "type": "string" + }, + "expression": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["last_edited_by"] + }, + "name": { + "type": "string" + }, + "last_edited_by": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["last_edited_time"] + }, + "name": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + } + } + }, { "type": "object", "properties": { @@ -101,42 +272,46 @@ }, "format": { "enum": [ - "number", - "number_with_commas", - "percent", - "dollar", + "argentine_peso", + "baht", "canadian_dollar", + "chilean_peso", + "colombian_peso", + "danish_krone", + "dirham", + "dollar", "euro", - "pound", - "yen", - "ruble", - "rupee", - "won", - "yuan", - "real", - "lira", - "rupiah", + "forint", "franc", "hong_kong_dollar", - "new_zealand_dollar", + "koruna", "krona", - "norwegian_krone", + "leu", + "lira", "mexican_peso", - "rand", "new_taiwan_dollar", - "danish_krone", - "zloty", - "baht", - "forint", - "koruna", - "shekel", - "chilean_peso", + "new_zealand_dollar", + "norwegian_krone", + "number", + "number_with_commas", + "percent", + "peruvian_sol", "philippine_peso", - "dirham", - "colombian_peso", - "riyal", + "pound", + "rand", + "real", "ringgit", - "leu" + "riyal", + "ruble", + "rupee", + "rupiah", + "shekel", + "singapore_dollar", + "uruguayan_peso", + "won", + "yen", + "yuan", + "zloty" ] } } @@ -168,16 +343,33 @@ "type": "string" }, "type": { - "enum": ["formula"] + "enum": ["people"] }, "name": { "type": "string" }, - "expression": { + "people": { "type": "string" } } }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["phone_number"] + }, + "name": { + "type": "string" + }, + "phone_number": { + "type": "object" + } + } + }, { "type": "object", "properties": { @@ -201,6 +393,23 @@ } } }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["rich_text"] + }, + "name": { + "type": "string" + }, + "rich_text": { + "type": "object" + } + } + }, { "type": "object", "properties": { @@ -227,23 +436,115 @@ }, "function": { "enum": [ - "count_all", - "count_values", - "count_unique_values", - "count_empty", - "count_not_empty", - "percent_empty", - "percent_not_empty", - "sum", "average", + "checked", + "count_per_group", + "count", + "count_values", + "date_range", + "earliest_date", + "empty", + "latest_date", + "max", "median", "min", - "max", + "not_empty", + "percent_checked", + "percent_empty", + "percent_not_empty", + "percent_per_group", + "percent_unchecked", "range", - "show_original" + "unchecked", + "unique", + "show_original", + "show_unique", + "sum" ] } } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["status"] + }, + "name": { + "type": "string" + }, + "status": { + "type": "object", + "properties": { + "options": { + "type": "array", + "items": { + "$ref": "options.json" + } + }, + "groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "color": { + "type": "string" + }, + "option_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["title"] + }, + "name": { + "type": "string" + }, + "title": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["url"] + }, + "name": { + "type": "string" + }, + "url": { + "type": "object" + } + } } ] } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json index 6152833eda6c..7972b07d6c73 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json @@ -10,14 +10,25 @@ "type": "string" }, "created_time": { - "type": "string" + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" }, "last_edited_time": { - "type": "string" + "type": "string", + "format": "date-time" + }, + "last_edited_by": { + "$ref": "user.json" }, "archived": { "type": "boolean" }, + "icon": { + "$ref": "icon.json" + }, "cover": { "$ref": "file.json" }, @@ -27,6 +38,9 @@ "url": { "type": "string" }, + "public_url": { + "type": ["null", "string"] + }, "properties": { "type": "array", "items": { @@ -52,18 +66,6 @@ "title": { "$ref": "title.json" } } }, - { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "enum": ["rich_text"] - }, - "rich_text": { "$ref": "rich_text.json" } - } - }, { "type": "object", "properties": { @@ -74,10 +76,8 @@ "enum": ["rich_text"] }, "rich_text": { - "type": ["array"], - "items": { - "$ref": "rich_text.json" - } + "type": ["null", "array"], + "items": { "$ref": "rich_text.json" } } } }, @@ -230,7 +230,7 @@ "properties": { "id": { "type": "string" }, "type": { "enum": ["phone_number"] }, - "phone_number": { "type": "string" } + "phone_number": { "type": "object" } } }, { @@ -264,6 +264,62 @@ "type": { "enum": ["last_edited_by"] }, "last_edited_by": { "$ref": "user.json" } } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["number"] }, + "number": { + "type": "object", + "properties": { + "format": { "type": "string" } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["status"] }, + "status": { "$ref": "options.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["unique_id"] }, + "unique_id": { + "type": "object", + "properties": { + "number": { "type": "number" }, + "prefix": { "type": ["null", "string"] } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["verification"] + }, + "verification": { + "type": "object", + "properties": { + "state": { + "enum": ["verified", "unverified"] + }, + "verified_by": { "$ref": "user.json" }, + "date": { "$ref": "date.json" } + } + } + } } ] } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json index eb526a17a235..b321541d43f6 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json @@ -4,6 +4,7 @@ "additionalProperties": true, "properties": { "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } } + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "is_toggleable": { "type": "boolean" } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json index 2043f0590a4a..db2cf20b9e81 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json @@ -4,22 +4,19 @@ "additionalProperties": true, "properties": { "type": { - "enum": ["database_id", "page_id", "workspace", "block_id"] + "enum": ["block_id", "database_id", "page_id", "workspace"] }, - "database_id": { + "block_id": { "type": "string" }, - "page_id": { + "database_id": { "type": "string" }, - "block_id": { + "page_id": { "type": "string" }, "workspace": { "type": "boolean" - }, - "block_id": { - "type": "string" } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json index 781c739a17be..cc003048b875 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json @@ -26,24 +26,21 @@ } } }, - "rich_text": { + "mention": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "content": { + "type": { + "type": ["null", "string"] + } + } + }, + "equation": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "expression": { "type": ["null", "string"] - }, - "link": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "type": { - "enum": ["url"] - }, - "url": { - "type": ["null", "string"] - } - } } } }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json index ee7ca504414e..65411d825b11 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json @@ -4,7 +4,7 @@ "additionalProperties": true, "properties": { "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "children": { "type": "array", "items": { "type": "object" } } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json index 2fb94bfa035e..d893b1147b7e 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json @@ -10,7 +10,7 @@ "type": "string" }, "name": { - "type": "string" + "type": ["null", "string"] }, "avatar_url": { "type": ["null", "string"] @@ -23,7 +23,7 @@ "additionalProperties": true, "properties": { "email": { - "type": "string" + "type": ["null", "string"] } } }, diff --git a/docs/integrations/sources/notion-migrations.md b/docs/integrations/sources/notion-migrations.md new file mode 100644 index 000000000000..d08cd0e331e0 --- /dev/null +++ b/docs/integrations/sources/notion-migrations.md @@ -0,0 +1,11 @@ +# Notion Migration Guide + +## Upgrading to 2.0.0 + +Version 2.0.0 introduces a number of changes to the JSON schemas of all streams. These changes are being introduced to reflect updates to the Notion API. Some breaking changes have been introduced that will affect the Blocks, Databases and Pages stream. + +- The type of the `rich_text` property in the Pages stream has been updated from an object to an array of `rich_text` objects +- The type of the `phone_number` property in the Pages stream has been updated from a string to an object +- The deprecated `text` property in content blocks has been renamed to `rich_text`. This change affects the Blocks, Databases and Pages streams. + +A full schema refresh and data reset are required when upgrading to this version. diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index e279c0b3133f..aa07862e0649 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -5,6 +5,7 @@ This page contains the setup guide and reference information for the Notion sour ## Prerequisites - Access to a [Notion](https://notion.so/login) workspace + ## Setup guide​ To authenticate the Notion source connector, you need to use **one** of the following two methods: @@ -105,6 +106,7 @@ The connector is restricted by Notion [request limits](https://developers.notion | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------- | +| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | | 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | | 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | | 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy |