From 2c4f3e17ca1c59bcab5c105581d600140b3971cb Mon Sep 17 00:00:00 2001 From: Ben Church Date: Fri, 13 Oct 2023 17:50:20 -0700 Subject: [PATCH] =?UTF-8?q?Revert=20"Connector=20Ops:=20Fix=20third=20part?= =?UTF-8?q?y=20support=20in=20get=5Fall=5Fconnectors=5F=E2=80=A6=20(#31413?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connector_ops/connector_ops/utils.py | 87 ++++--------------- .../connector_ops/tests/test_utils.py | 7 +- .../pipelines/commands/groups/connectors.py | 3 +- .../{appstore.md => appstore-singer.md} | 0 docusaurus/redirects.yml | 4 +- 5 files changed, 21 insertions(+), 80 deletions(-) rename docs/integrations/sources/{appstore.md => appstore-singer.md} (100%) diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 67126526dfcf..0f24ed63ba34 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -24,7 +24,6 @@ DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" -BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-" DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-" @@ -252,8 +251,7 @@ class ConnectorLanguageError(Exception): class Connector: """Utility class to gather metadata about a connector.""" - # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source - relative_connector_path: str + technical_name: str def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: if "-" not in self.technical_name: @@ -262,70 +260,35 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: name = self.technical_name[len(_type) + 1 :] return _type, name - @property - def technical_name(self) -> str: - """ - Return the technical name of the connector from the given relative_connector_path - e.g. source-google-sheets -> source-google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source - """ - return self.relative_connector_path.split("/")[-1] - @property def name(self): return self._get_type_and_name_from_technical_name()[1] @property def connector_type(self) -> str: - return self.metadata["connectorType"] - - @property - def is_third_party(self) -> bool: - return THIRD_PARTY_GLOB in self.relative_connector_path - - @property - def has_airbyte_docs(self) -> bool: - return ( - self.metadata - and self.metadata.get("documentationUrl") is not None - and BASE_AIRBYTE_DOCS_URL in self.metadata.get("documentationUrl") - ) + return self._get_type_and_name_from_technical_name()[0] @property def documentation_directory(self) -> Path: - if not self.has_airbyte_docs: - return None return Path(f"./docs/integrations/{self.connector_type}s") - @property - def relative_documentation_path_str(self) -> str: - documentation_url = self.metadata["documentationUrl"] - relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "") - - # strip leading and trailing slashes - relative_documentation_path = relative_documentation_path.strip("/") - - return f"./docs/{relative_documentation_path}" - @property def documentation_file_path(self) -> Path: - if not self.has_airbyte_docs: - return None - - return Path(f"{self.relative_documentation_path_str}.md") + readme_file_name = f"{self.name}.md" + return self.documentation_directory / readme_file_name @property def inapp_documentation_file_path(self) -> Path: - if not self.has_airbyte_docs: - return None + readme_file_name = f"{self.name}.inapp.md" + return self.documentation_directory / readme_file_name - return Path(f"{self.relative_documentation_path_str}.inapp.md") + @property + def migration_guide_file_name(self) -> str: + return f"{self.name}-migrations.md" @property def migration_guide_file_path(self) -> Path: - if not self.has_airbyte_docs: - return None - - return Path(f"{self.relative_documentation_path_str}-migrations.md") + return self.documentation_directory / self.migration_guide_file_name @property def icon_path(self) -> Path: @@ -334,7 +297,7 @@ def icon_path(self) -> Path: @property def code_directory(self) -> Path: - return Path(f"./{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}") + return Path(f"./airbyte-integrations/connectors/{self.technical_name}") @property def has_dockerfile(self) -> bool: @@ -573,26 +536,6 @@ def get_changed_connectors( return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files} -def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str: - """Get the relative connector folder name from the metadata file path. - - Args: - metadata_file_path (Path): Path to the metadata file. - - Returns: - str: The relative connector folder name. - """ - # remove CONNECTOR_PATH_PREFIX and anything before - metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1] - - # remove metadata.yaml - metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "") - - # remove leading and trailing slashes - metadata_file_path = metadata_file_path.strip("/") - return metadata_file_path - - def get_all_connectors_in_repo() -> Set[Connector]: """Retrieve a set of all Connectors in the repo. We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name. @@ -604,9 +547,11 @@ def get_all_connectors_in_repo() -> Set[Connector]: repo_path = repo.working_tree_dir return { - Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file)) - for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True) - if SCAFFOLD_CONNECTOR_GLOB not in metadata_file + Connector(Path(metadata_file).parent.name) + for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) + # HACK: The Connector util is not good at fetching metadata for third party connectors. + # We want to avoid picking a connector that does not have metadata. + if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file } diff --git a/airbyte-ci/connectors/connector_ops/tests/test_utils.py b/airbyte-ci/connectors/connector_ops/tests/test_utils.py index fe74983ef010..f8d61cd3ba2e 100644 --- a/airbyte-ci/connectors/connector_ops/tests/test_utils.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_utils.py @@ -176,8 +176,5 @@ def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies): def test_get_all_connectors_in_repo(): all_connectors = utils.get_all_connectors_in_repo() assert len(all_connectors) > 0 - for connector in all_connectors: - assert isinstance(connector, utils.Connector) - assert connector.metadata is not None - if connector.has_airbyte_docs: - assert connector.documentation_file_path.exists() + assert all([isinstance(connector, utils.Connector) for connector in all_connectors]) + assert all([connector.metadata is not None for connector in all_connectors]) diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py index 30d06cd20ca0..b4b8e0d7711e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py @@ -103,8 +103,7 @@ def get_selected_connectors_with_modified_files( selected_connectors_with_modified_files = [] for connector in selected_connectors: connector_with_modified_files = ConnectorWithModifiedFiles( - relative_connector_path=connector.relative_connector_path, - modified_files=get_connector_modified_files(connector, modified_files), + technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files) ) if not metadata_changes_only: selected_connectors_with_modified_files.append(connector_with_modified_files) diff --git a/docs/integrations/sources/appstore.md b/docs/integrations/sources/appstore-singer.md similarity index 100% rename from docs/integrations/sources/appstore.md rename to docs/integrations/sources/appstore-singer.md diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml index b69386db8c1d..1014d9a7758c 100644 --- a/docusaurus/redirects.yml +++ b/docusaurus/redirects.yml @@ -5,8 +5,8 @@ to: /operator-guides/upgrading-airbyte - from: /catalog to: /understanding-airbyte/airbyte-protocol -- from: /integrations/sources/appstore-singer - to: /integrations/sources/appstore +- from: /integrations/sources/appstore + to: /integrations/sources/appstore-singer - from: - /project-overview/security - /operator-guides/securing-airbyte