diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 86c762510cc5..c6d83f700882 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -24,6 +24,7 @@ DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" +BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-" DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-" @@ -251,7 +252,8 @@ class ConnectorLanguageError(Exception): class Connector: """Utility class to gather metadata about a connector.""" - technical_name: str + # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source + _relative_connector_path: str def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: if "-" not in self.technical_name: @@ -260,35 +262,53 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: name = self.technical_name[len(_type) + 1 :] return _type, name + @property + def technical_name(self) -> str: + """ + Return the technical name of the connector from the given _relative_connector_path + e.g. source-google-sheets -> google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source + """ + return self._relative_connector_path.split("/")[-1] + @property def name(self): return self._get_type_and_name_from_technical_name()[1] @property def connector_type(self) -> str: - return self._get_type_and_name_from_technical_name()[0] + return self.metadata["connectorType"] + + @property + def is_third_party(self) -> bool: + return THIRD_PARTY_GLOB in self._relative_connector_path @property def documentation_directory(self) -> Path: + if self.has_external_documentation: + return None return Path(f"./docs/integrations/{self.connector_type}s") @property - def documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.md" - return self.documentation_directory / readme_file_name + def relative_documentation_path_str(self) -> str: + documentation_url = self.metadata["documentationUrl"] + relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "") + + # strip leading and trailing slashes + relative_documentation_path = relative_documentation_path.strip("/") + + return f"./docs/{relative_documentation_path}" @property - def inapp_documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.inapp.md" - return self.documentation_directory / readme_file_name + def documentation_file_path(self) -> Path: + return Path(f"{self.relative_documentation_path_str}.md") @property - def migration_guide_file_name(self) -> str: - return f"{self.name}-migrations.md" + def inapp_documentation_file_path(self) -> Path: + return Path(f"{self.relative_documentation_path_str}.inapp.md") @property def migration_guide_file_path(self) -> Path: - return self.documentation_directory / self.migration_guide_file_name + return Path(f"{self.relative_documentation_path_str}-migrations.md") @property def icon_path(self) -> Path: @@ -297,7 +317,7 @@ def icon_path(self) -> Path: @property def code_directory(self) -> Path: - return Path(f"./airbyte-integrations/connectors/{self.technical_name}") + return Path(f"./{CONNECTOR_PATH_PREFIX}/{self._relative_connector_path}") @property def metadata_file_path(self) -> Path: @@ -529,6 +549,26 @@ def get_changed_connectors( return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files} +def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str: + """Get the relative connector folder name from the metadata file path. + + Args: + metadata_file_path (Path): Path to the metadata file. + + Returns: + str: The relative connector folder name. + """ + # remove CONNECTOR_PATH_PREFIX and anything before + metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1] + + # remove metadata.yaml + metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "") + + # remove leading and trailing slashes + metadata_file_path = metadata_file_path.strip("/") + return metadata_file_path + + def get_all_connectors_in_repo() -> Set[Connector]: """Retrieve a set of all Connectors in the repo. We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name. @@ -540,11 +580,9 @@ def get_all_connectors_in_repo() -> Set[Connector]: repo_path = repo.working_tree_dir return { - Connector(Path(metadata_file).parent.name) - for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) - # HACK: The Connector util is not good at fetching metadata for third party connectors. - # We want to avoid picking a connector that does not have metadata. - if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file + Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file)) + for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True) + if SCAFFOLD_CONNECTOR_GLOB not in metadata_file } diff --git a/airbyte-ci/connectors/connector_ops/pyproject.toml b/airbyte-ci/connectors/connector_ops/pyproject.toml index 4c245651c2d3..f33f34b457f4 100644 --- a/airbyte-ci/connectors/connector_ops/pyproject.toml +++ b/airbyte-ci/connectors/connector_ops/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector_ops" -version = "0.2.5" +version = "0.2.6" description = "Packaged maintained by the connector operations team to perform CI for connectors" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/connector_ops/tests/test_utils.py b/airbyte-ci/connectors/connector_ops/tests/test_utils.py index 5595386945f8..3a7c82ee537a 100644 --- a/airbyte-ci/connectors/connector_ops/tests/test_utils.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_utils.py @@ -199,5 +199,7 @@ def test_get_all_gradle_dependencies(with_test_dependencies): def test_get_all_connectors_in_repo(): all_connectors = utils.get_all_connectors_in_repo() assert len(all_connectors) > 0 - assert all([isinstance(connector, utils.Connector) for connector in all_connectors]) - assert all([connector.metadata is not None for connector in all_connectors]) + for connector in all_connectors: + assert isinstance(connector, utils.Connector) + assert connector.metadata is not None + assert connector.documentation_file_path.exists() diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index ca445f3e79ee..3786009d1a7c 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -380,7 +380,7 @@ url = "../common_utils" [[package]] name = "connector-ops" -version = "0.2.5" +version = "0.2.6" description = "Packaged maintained by the connector operations team to perform CI for connectors" optional = false python-versions = "^3.10" diff --git a/docs/integrations/sources/appstore-singer.md b/docs/integrations/sources/appstore.md similarity index 100% rename from docs/integrations/sources/appstore-singer.md rename to docs/integrations/sources/appstore.md diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml index 1014d9a7758c..b69386db8c1d 100644 --- a/docusaurus/redirects.yml +++ b/docusaurus/redirects.yml @@ -5,8 +5,8 @@ to: /operator-guides/upgrading-airbyte - from: /catalog to: /understanding-airbyte/airbyte-protocol -- from: /integrations/sources/appstore - to: /integrations/sources/appstore-singer +- from: /integrations/sources/appstore-singer + to: /integrations/sources/appstore - from: - /project-overview/security - /operator-guides/securing-airbyte