Revert "Connector Ops: Fix third party support in get_all_connectors_… (

#31413)

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -24,7 +24,6 @@
 
 DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master")
 OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
-BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com"
 CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
 SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-"
 DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-"
@@ -252,8 +251,7 @@ class ConnectorLanguageError(Exception):
 class Connector:
     """Utility class to gather metadata about a connector."""
 
-    # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source
-    relative_connector_path: str
+    technical_name: str
 
     def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]:
         if "-" not in self.technical_name:
@@ -262,70 +260,35 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]:
         name = self.technical_name[len(_type) + 1 :]
         return _type, name
 
-    @property
-    def technical_name(self) -> str:
-        """
-        Return the technical name of the connector from the given relative_connector_path
-        e.g. source-google-sheets -> source-google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source
-        """
-        return self.relative_connector_path.split("/")[-1]
-
     @property
     def name(self):
         return self._get_type_and_name_from_technical_name()[1]
 
     @property
     def connector_type(self) -> str:
-        return self.metadata["connectorType"]
-
-    @property
-    def is_third_party(self) -> bool:
-        return THIRD_PARTY_GLOB in self.relative_connector_path
-
-    @property
-    def has_airbyte_docs(self) -> bool:
-        return (
-            self.metadata
-            and self.metadata.get("documentationUrl") is not None
-            and BASE_AIRBYTE_DOCS_URL in self.metadata.get("documentationUrl")
-        )
+        return self._get_type_and_name_from_technical_name()[0]
 
     @property
     def documentation_directory(self) -> Path:
-        if not self.has_airbyte_docs:
-            return None
         return Path(f"./docs/integrations/{self.connector_type}s")
 
-    @property
-    def relative_documentation_path_str(self) -> str:
-        documentation_url = self.metadata["documentationUrl"]
-        relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "")
-
-        # strip leading and trailing slashes
-        relative_documentation_path = relative_documentation_path.strip("/")
-
-        return f"./docs/{relative_documentation_path}"
-
     @property
     def documentation_file_path(self) -> Path:
-        if not self.has_airbyte_docs:
-            return None
-
-        return Path(f"{self.relative_documentation_path_str}.md")
+        readme_file_name = f"{self.name}.md"
+        return self.documentation_directory / readme_file_name
 
     @property
     def inapp_documentation_file_path(self) -> Path:
-        if not self.has_airbyte_docs:
-            return None
+        readme_file_name = f"{self.name}.inapp.md"
+        return self.documentation_directory / readme_file_name
 
-        return Path(f"{self.relative_documentation_path_str}.inapp.md")
+    @property
+    def migration_guide_file_name(self) -> str:
+        return f"{self.name}-migrations.md"
 
     @property
     def migration_guide_file_path(self) -> Path:
-        if not self.has_airbyte_docs:
-            return None
-
-        return Path(f"{self.relative_documentation_path_str}-migrations.md")
+        return self.documentation_directory / self.migration_guide_file_name
 
     @property
     def icon_path(self) -> Path:
@@ -334,7 +297,7 @@ def icon_path(self) -> Path:
 
     @property
     def code_directory(self) -> Path:
-        return Path(f"./{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}")
+        return Path(f"./airbyte-integrations/connectors/{self.technical_name}")
 
     @property
     def has_dockerfile(self) -> bool:
@@ -573,26 +536,6 @@ def get_changed_connectors(
     return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files}
 
 
-def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str:
-    """Get the relative connector folder name from the metadata file path.
-
-    Args:
-        metadata_file_path (Path): Path to the metadata file.
-
-    Returns:
-        str: The relative connector folder name.
-    """
-    # remove CONNECTOR_PATH_PREFIX and anything before
-    metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1]
-
-    # remove metadata.yaml
-    metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "")
-
-    # remove leading and trailing slashes
-    metadata_file_path = metadata_file_path.strip("/")
-    return metadata_file_path
-
-
 def get_all_connectors_in_repo() -> Set[Connector]:
     """Retrieve a set of all Connectors in the repo.
     We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name.
@@ -604,9 +547,11 @@ def get_all_connectors_in_repo() -> Set[Connector]:
     repo_path = repo.working_tree_dir
 
     return {
-        Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file))
-        for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True)
-        if SCAFFOLD_CONNECTOR_GLOB not in metadata_file
+        Connector(Path(metadata_file).parent.name)
+        for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
+        # HACK: The Connector util is not good at fetching metadata for third party connectors.
+        # We want to avoid picking a connector that does not have metadata.
+        if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file
     }
 
 

airbyte-ci/connectors/connector_ops/tests/test_utils.py

@@ -176,8 +176,5 @@ def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies):
 def test_get_all_connectors_in_repo():
     all_connectors = utils.get_all_connectors_in_repo()
     assert len(all_connectors) > 0
-    for connector in all_connectors:
-        assert isinstance(connector, utils.Connector)
-        assert connector.metadata is not None
-        if connector.has_airbyte_docs:
-            assert connector.documentation_file_path.exists()
+    assert all([isinstance(connector, utils.Connector) for connector in all_connectors])
+    assert all([connector.metadata is not None for connector in all_connectors])

airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py

@@ -103,8 +103,7 @@ def get_selected_connectors_with_modified_files(
     selected_connectors_with_modified_files = []
     for connector in selected_connectors:
         connector_with_modified_files = ConnectorWithModifiedFiles(
-            relative_connector_path=connector.relative_connector_path,
-            modified_files=get_connector_modified_files(connector, modified_files),
+            technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files)
         )
         if not metadata_changes_only:
             selected_connectors_with_modified_files.append(connector_with_modified_files)

docusaurus/redirects.yml

@@ -5,8 +5,8 @@
   to: /operator-guides/upgrading-airbyte
 - from: /catalog
   to: /understanding-airbyte/airbyte-protocol
-- from: /integrations/sources/appstore-singer
-  to: /integrations/sources/appstore
+- from: /integrations/sources/appstore
+  to: /integrations/sources/appstore-singer
 - from:
     - /project-overview/security
     - /operator-guides/securing-airbyte