Connector Ops: Fix third party support in get_all_connectors_in_repo (a…

…irbytehq#31324)

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -24,6 +24,7 @@
 
 DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master")
 OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
+BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com"
 CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
 SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-"
 DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-"
@@ -251,7 +252,8 @@ class ConnectorLanguageError(Exception):
 class Connector:
     """Utility class to gather metadata about a connector."""
 
-    technical_name: str
+    # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source
+    relative_connector_path: str
 
     def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]:
         if "-" not in self.technical_name:
@@ -260,35 +262,70 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]:
         name = self.technical_name[len(_type) + 1 :]
         return _type, name
 
+    @property
+    def technical_name(self) -> str:
+        """
+        Return the technical name of the connector from the given relative_connector_path
+        e.g. source-google-sheets -> source-google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source
+        """
+        return self.relative_connector_path.split("/")[-1]
+
     @property
     def name(self):
         return self._get_type_and_name_from_technical_name()[1]
 
     @property
     def connector_type(self) -> str:
-        return self._get_type_and_name_from_technical_name()[0]
+        return self.metadata["connectorType"]
+
+    @property
+    def is_third_party(self) -> bool:
+        return THIRD_PARTY_GLOB in self.relative_connector_path
+
+    @property
+    def has_airbyte_docs(self) -> bool:
+        return (
+            self.metadata
+            and self.metadata.get("documentationUrl") is not None
+            and BASE_AIRBYTE_DOCS_URL in self.metadata.get("documentationUrl")
+        )
 
     @property
     def documentation_directory(self) -> Path:
+        if not self.has_airbyte_docs:
+            return None
         return Path(f"./docs/integrations/{self.connector_type}s")
 
+    @property
+    def relative_documentation_path_str(self) -> str:
+        documentation_url = self.metadata["documentationUrl"]
+        relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "")
+
+        # strip leading and trailing slashes
+        relative_documentation_path = relative_documentation_path.strip("/")
+
+        return f"./docs/{relative_documentation_path}"
+
     @property
     def documentation_file_path(self) -> Path:
-        readme_file_name = f"{self.name}.md"
-        return self.documentation_directory / readme_file_name
+        if not self.has_airbyte_docs:
+            return None
+
+        return Path(f"{self.relative_documentation_path_str}.md")
 
     @property
     def inapp_documentation_file_path(self) -> Path:
-        readme_file_name = f"{self.name}.inapp.md"
-        return self.documentation_directory / readme_file_name
+        if not self.has_airbyte_docs:
+            return None
 
-    @property
-    def migration_guide_file_name(self) -> str:
-        return f"{self.name}-migrations.md"
+        return Path(f"{self.relative_documentation_path_str}.inapp.md")
 
     @property
     def migration_guide_file_path(self) -> Path:
-        return self.documentation_directory / self.migration_guide_file_name
+        if not self.has_airbyte_docs:
+            return None
+
+        return Path(f"{self.relative_documentation_path_str}-migrations.md")
 
     @property
     def icon_path(self) -> Path:
@@ -297,7 +334,7 @@ def icon_path(self) -> Path:
 
     @property
     def code_directory(self) -> Path:
-        return Path(f"./airbyte-integrations/connectors/{self.technical_name}")
+        return Path(f"./{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}")
 
     @property
     def has_dockerfile(self) -> bool:
@@ -536,6 +573,26 @@ def get_changed_connectors(
     return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files}
 
 
+def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str:
+    """Get the relative connector folder name from the metadata file path.
+
+    Args:
+        metadata_file_path (Path): Path to the metadata file.
+
+    Returns:
+        str: The relative connector folder name.
+    """
+    # remove CONNECTOR_PATH_PREFIX and anything before
+    metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1]
+
+    # remove metadata.yaml
+    metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "")
+
+    # remove leading and trailing slashes
+    metadata_file_path = metadata_file_path.strip("/")
+    return metadata_file_path
+
+
 def get_all_connectors_in_repo() -> Set[Connector]:
     """Retrieve a set of all Connectors in the repo.
     We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name.
@@ -547,11 +604,9 @@ def get_all_connectors_in_repo() -> Set[Connector]:
     repo_path = repo.working_tree_dir
 
     return {
-        Connector(Path(metadata_file).parent.name)
-        for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
-        # HACK: The Connector util is not good at fetching metadata for third party connectors.
-        # We want to avoid picking a connector that does not have metadata.
-        if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file
+        Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file))
+        for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True)
+        if SCAFFOLD_CONNECTOR_GLOB not in metadata_file
     }
 
 

airbyte-ci/connectors/connector_ops/tests/test_utils.py

@@ -176,5 +176,8 @@ def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies):
 def test_get_all_connectors_in_repo():
     all_connectors = utils.get_all_connectors_in_repo()
     assert len(all_connectors) > 0
-    assert all([isinstance(connector, utils.Connector) for connector in all_connectors])
-    assert all([connector.metadata is not None for connector in all_connectors])
+    for connector in all_connectors:
+        assert isinstance(connector, utils.Connector)
+        assert connector.metadata is not None
+        if connector.has_airbyte_docs:
+            assert connector.documentation_file_path.exists()

airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py

@@ -103,7 +103,8 @@ def get_selected_connectors_with_modified_files(
     selected_connectors_with_modified_files = []
     for connector in selected_connectors:
         connector_with_modified_files = ConnectorWithModifiedFiles(
-            technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files)
+            relative_connector_path=connector.relative_connector_path,
+            modified_files=get_connector_modified_files(connector, modified_files),
         )
         if not metadata_changes_only:
             selected_connectors_with_modified_files.append(connector_with_modified_files)

docusaurus/redirects.yml

@@ -5,8 +5,8 @@
   to: /operator-guides/upgrading-airbyte
 - from: /catalog
   to: /understanding-airbyte/airbyte-protocol
-- from: /integrations/sources/appstore
-  to: /integrations/sources/appstore-singer
+- from: /integrations/sources/appstore-singer
+  to: /integrations/sources/appstore
 - from:
     - /project-overview/security
     - /operator-guides/securing-airbyte