diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 0f24ed63ba34..67126526dfcf 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -24,6 +24,7 @@ DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" +BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-" DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-" @@ -251,7 +252,8 @@ class ConnectorLanguageError(Exception): class Connector: """Utility class to gather metadata about a connector.""" - technical_name: str + # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source + relative_connector_path: str def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: if "-" not in self.technical_name: @@ -260,35 +262,70 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: name = self.technical_name[len(_type) + 1 :] return _type, name + @property + def technical_name(self) -> str: + """ + Return the technical name of the connector from the given relative_connector_path + e.g. source-google-sheets -> source-google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source + """ + return self.relative_connector_path.split("/")[-1] + @property def name(self): return self._get_type_and_name_from_technical_name()[1] @property def connector_type(self) -> str: - return self._get_type_and_name_from_technical_name()[0] + return self.metadata["connectorType"] + + @property + def is_third_party(self) -> bool: + return THIRD_PARTY_GLOB in self.relative_connector_path + + @property + def has_airbyte_docs(self) -> bool: + return ( + self.metadata + and self.metadata.get("documentationUrl") is not None + and BASE_AIRBYTE_DOCS_URL in self.metadata.get("documentationUrl") + ) @property def documentation_directory(self) -> Path: + if not self.has_airbyte_docs: + return None return Path(f"./docs/integrations/{self.connector_type}s") + @property + def relative_documentation_path_str(self) -> str: + documentation_url = self.metadata["documentationUrl"] + relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "") + + # strip leading and trailing slashes + relative_documentation_path = relative_documentation_path.strip("/") + + return f"./docs/{relative_documentation_path}" + @property def documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.md" - return self.documentation_directory / readme_file_name + if not self.has_airbyte_docs: + return None + + return Path(f"{self.relative_documentation_path_str}.md") @property def inapp_documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.inapp.md" - return self.documentation_directory / readme_file_name + if not self.has_airbyte_docs: + return None - @property - def migration_guide_file_name(self) -> str: - return f"{self.name}-migrations.md" + return Path(f"{self.relative_documentation_path_str}.inapp.md") @property def migration_guide_file_path(self) -> Path: - return self.documentation_directory / self.migration_guide_file_name + if not self.has_airbyte_docs: + return None + + return Path(f"{self.relative_documentation_path_str}-migrations.md") @property def icon_path(self) -> Path: @@ -297,7 +334,7 @@ def icon_path(self) -> Path: @property def code_directory(self) -> Path: - return Path(f"./airbyte-integrations/connectors/{self.technical_name}") + return Path(f"./{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}") @property def has_dockerfile(self) -> bool: @@ -536,6 +573,26 @@ def get_changed_connectors( return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files} +def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str: + """Get the relative connector folder name from the metadata file path. + + Args: + metadata_file_path (Path): Path to the metadata file. + + Returns: + str: The relative connector folder name. + """ + # remove CONNECTOR_PATH_PREFIX and anything before + metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1] + + # remove metadata.yaml + metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "") + + # remove leading and trailing slashes + metadata_file_path = metadata_file_path.strip("/") + return metadata_file_path + + def get_all_connectors_in_repo() -> Set[Connector]: """Retrieve a set of all Connectors in the repo. We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name. @@ -547,11 +604,9 @@ def get_all_connectors_in_repo() -> Set[Connector]: repo_path = repo.working_tree_dir return { - Connector(Path(metadata_file).parent.name) - for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) - # HACK: The Connector util is not good at fetching metadata for third party connectors. - # We want to avoid picking a connector that does not have metadata. - if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file + Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file)) + for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True) + if SCAFFOLD_CONNECTOR_GLOB not in metadata_file } diff --git a/airbyte-ci/connectors/connector_ops/tests/test_utils.py b/airbyte-ci/connectors/connector_ops/tests/test_utils.py index f8d61cd3ba2e..fe74983ef010 100644 --- a/airbyte-ci/connectors/connector_ops/tests/test_utils.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_utils.py @@ -176,5 +176,8 @@ def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies): def test_get_all_connectors_in_repo(): all_connectors = utils.get_all_connectors_in_repo() assert len(all_connectors) > 0 - assert all([isinstance(connector, utils.Connector) for connector in all_connectors]) - assert all([connector.metadata is not None for connector in all_connectors]) + for connector in all_connectors: + assert isinstance(connector, utils.Connector) + assert connector.metadata is not None + if connector.has_airbyte_docs: + assert connector.documentation_file_path.exists() diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py index b4b8e0d7711e..30d06cd20ca0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py @@ -103,7 +103,8 @@ def get_selected_connectors_with_modified_files( selected_connectors_with_modified_files = [] for connector in selected_connectors: connector_with_modified_files = ConnectorWithModifiedFiles( - technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files) + relative_connector_path=connector.relative_connector_path, + modified_files=get_connector_modified_files(connector, modified_files), ) if not metadata_changes_only: selected_connectors_with_modified_files.append(connector_with_modified_files) diff --git a/docs/integrations/sources/appstore-singer.md b/docs/integrations/sources/appstore.md similarity index 100% rename from docs/integrations/sources/appstore-singer.md rename to docs/integrations/sources/appstore.md diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml index 1014d9a7758c..b69386db8c1d 100644 --- a/docusaurus/redirects.yml +++ b/docusaurus/redirects.yml @@ -5,8 +5,8 @@ to: /operator-guides/upgrading-airbyte - from: /catalog to: /understanding-airbyte/airbyte-protocol -- from: /integrations/sources/appstore - to: /integrations/sources/appstore-singer +- from: /integrations/sources/appstore-singer + to: /integrations/sources/appstore - from: - /project-overview/security - /operator-guides/securing-airbyte