From 625a4b17149c252ec8a57de0e43fdf9806963c44 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 11 Oct 2023 15:18:06 -0700 Subject: [PATCH] Connector Ops: Fix third party support in get_all_connectors_in_repo (#31166) Co-authored-by: bnchrch --- .../connector_ops/connector_ops/utils.py | 72 ++++++++++++++----- .../connector_ops/tests/test_utils.py | 6 +- .../{appstore-singer.md => appstore.md} | 0 docusaurus/redirects.yml | 4 +- 4 files changed, 61 insertions(+), 21 deletions(-) rename docs/integrations/sources/{appstore-singer.md => appstore.md} (100%) diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 0f24ed63ba34..09e399002021 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -24,6 +24,7 @@ DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" +BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-" DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-" @@ -251,7 +252,8 @@ class ConnectorLanguageError(Exception): class Connector: """Utility class to gather metadata about a connector.""" - technical_name: str + # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source + _relative_connector_path: str def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: if "-" not in self.technical_name: @@ -260,35 +262,53 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: name = self.technical_name[len(_type) + 1 :] return _type, name + @property + def technical_name(self) -> str: + """ + Return the technical name of the connector from the given _relative_connector_path + e.g. source-google-sheets -> google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source + """ + return self._relative_connector_path.split("/")[-1] + @property def name(self): return self._get_type_and_name_from_technical_name()[1] @property def connector_type(self) -> str: - return self._get_type_and_name_from_technical_name()[0] + return self.metadata["connectorType"] + + @property + def is_third_party(self) -> bool: + return THIRD_PARTY_GLOB in self._relative_connector_path @property def documentation_directory(self) -> Path: + if self.has_external_documentation: + return None return Path(f"./docs/integrations/{self.connector_type}s") @property - def documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.md" - return self.documentation_directory / readme_file_name + def relative_documentation_path_str(self) -> str: + documentation_url = self.metadata["documentationUrl"] + relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "") + + # strip leading and trailing slashes + relative_documentation_path = relative_documentation_path.strip("/") + + return f"./docs/{relative_documentation_path}" @property - def inapp_documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.inapp.md" - return self.documentation_directory / readme_file_name + def documentation_file_path(self) -> Path: + return Path(f"{self.relative_documentation_path_str}.md") @property - def migration_guide_file_name(self) -> str: - return f"{self.name}-migrations.md" + def inapp_documentation_file_path(self) -> Path: + return Path(f"{self.relative_documentation_path_str}.inapp.md") @property def migration_guide_file_path(self) -> Path: - return self.documentation_directory / self.migration_guide_file_name + return Path(f"{self.relative_documentation_path_str}-migrations.md") @property def icon_path(self) -> Path: @@ -297,7 +317,7 @@ def icon_path(self) -> Path: @property def code_directory(self) -> Path: - return Path(f"./airbyte-integrations/connectors/{self.technical_name}") + return Path(f"./{CONNECTOR_PATH_PREFIX}/{self._relative_connector_path}") @property def has_dockerfile(self) -> bool: @@ -536,6 +556,26 @@ def get_changed_connectors( return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files} +def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str: + """Get the relative connector folder name from the metadata file path. + + Args: + metadata_file_path (Path): Path to the metadata file. + + Returns: + str: The relative connector folder name. + """ + # remove CONNECTOR_PATH_PREFIX and anything before + metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1] + + # remove metadata.yaml + metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "") + + # remove leading and trailing slashes + metadata_file_path = metadata_file_path.strip("/") + return metadata_file_path + + def get_all_connectors_in_repo() -> Set[Connector]: """Retrieve a set of all Connectors in the repo. We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name. @@ -547,11 +587,9 @@ def get_all_connectors_in_repo() -> Set[Connector]: repo_path = repo.working_tree_dir return { - Connector(Path(metadata_file).parent.name) - for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) - # HACK: The Connector util is not good at fetching metadata for third party connectors. - # We want to avoid picking a connector that does not have metadata. - if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file + Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file)) + for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True) + if SCAFFOLD_CONNECTOR_GLOB not in metadata_file } diff --git a/airbyte-ci/connectors/connector_ops/tests/test_utils.py b/airbyte-ci/connectors/connector_ops/tests/test_utils.py index 5b3eecf016f8..4bbb521b92fd 100644 --- a/airbyte-ci/connectors/connector_ops/tests/test_utils.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_utils.py @@ -179,5 +179,7 @@ def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies): def test_get_all_connectors_in_repo(): all_connectors = utils.get_all_connectors_in_repo() assert len(all_connectors) > 0 - assert all([isinstance(connector, utils.Connector) for connector in all_connectors]) - assert all([connector.metadata is not None for connector in all_connectors]) + for connector in all_connectors: + assert isinstance(connector, utils.Connector) + assert connector.metadata is not None + assert connector.documentation_file_path.exists() diff --git a/docs/integrations/sources/appstore-singer.md b/docs/integrations/sources/appstore.md similarity index 100% rename from docs/integrations/sources/appstore-singer.md rename to docs/integrations/sources/appstore.md diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml index 1014d9a7758c..b69386db8c1d 100644 --- a/docusaurus/redirects.yml +++ b/docusaurus/redirects.yml @@ -5,8 +5,8 @@ to: /operator-guides/upgrading-airbyte - from: /catalog to: /understanding-airbyte/airbyte-protocol -- from: /integrations/sources/appstore - to: /integrations/sources/appstore-singer +- from: /integrations/sources/appstore-singer + to: /integrations/sources/appstore - from: - /project-overview/security - /operator-guides/securing-airbyte