airbyte-ci: Use absolute paths for Connector

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -24,6 +24,7 @@
 
 DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master")
 OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
+BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com"
 CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
 SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-"
 DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-"
@@ -32,7 +33,6 @@
 THIRD_PARTY_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + f"/{THIRD_PARTY_GLOB}/"
 SCAFFOLD_CONNECTOR_GLOB = "-scaffold-"
 
-
 ACCEPTANCE_TEST_CONFIG_FILE_NAME = "acceptance-test-config.yml"
 AIRBYTE_DOCKER_REPO = "airbyte"
 AIRBYTE_REPO_DIRECTORY_NAME = "airbyte"
@@ -66,6 +66,39 @@ def download_catalog(catalog_url):
 }
 
 
+def get_airbyte_repo() -> git.Repo:
+    """Get the airbyte repo."""
+    return git.Repo(search_parent_directories=True)
+
+
+@functools.lru_cache(maxsize=1)
+def get_airbyte_repo_path_with_fallback() -> Path:
+    """Get the airbyte repo path."""
+    try:
+        return get_airbyte_repo().working_tree_dir
+    except git.exc.InvalidGitRepositoryError:
+        logging.warning("Could not find the airbyte repo, falling back to the current working directory.")
+        path = Path.cwd()
+        logging.warning(f"Using {path} as the airbyte repo path.")
+        # log the files in the current directory to help debug
+        logging.warning(f"Files in {path}: {os.listdir(path)}")
+        return path
+
+
+def abs_project_path_to_relative_path_str(absolute_path: Path) -> str:
+    """Get the relative path from the absolute path.
+
+    e.g. /Users/airbyte/airbyte/integrations/source-google-sheets -> integrations/source-google-sheets
+
+    Args:
+        absolute_path (Path): The absolute path.
+
+    Returns:
+        str: The relative path string.
+    """
+    return str(absolute_path).replace(get_airbyte_repo_path_with_fallback(), "").strip("/")
+
+
 class ConnectorInvalidNameError(Exception):
     pass
 
@@ -87,7 +120,6 @@ def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[
     Returns:
         Set[Connector]: Set of connectors that were changed
     """
-    airbyte_repo = git.Repo(search_parent_directories=True)
 
     if diff_regex is None:
         diff_command_args = ("--name-only", DIFFED_BRANCH)
@@ -96,7 +128,7 @@ def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[
 
     changed_acceptance_test_config_paths = {
         file_path
-        for file_path in airbyte_repo.git.diff(*diff_command_args).split("\n")
+        for file_path in get_airbyte_repo().git.diff(*diff_command_args).split("\n")
         if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX) and file_path.endswith(ACCEPTANCE_TEST_CONFIG_FILE_NAME)
     }
     return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_acceptance_test_config_paths}
@@ -251,7 +283,8 @@ class ConnectorLanguageError(Exception):
 class Connector:
     """Utility class to gather metadata about a connector."""
 
-    technical_name: str
+    # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source
+    relative_connector_path: str
 
     def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]:
         if "-" not in self.technical_name:
@@ -260,35 +293,77 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]:
         name = self.technical_name[len(_type) + 1 :]
         return _type, name
 
+    @property
+    def technical_name(self) -> str:
+        """
+        Return the technical name of the connector from the given relative_connector_path
+        e.g. source-google-sheets -> source-google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source
+        """
+        return self.relative_connector_path.split("/")[-1]
+
     @property
     def name(self):
         return self._get_type_and_name_from_technical_name()[1]
 
     @property
     def connector_type(self) -> str:
-        return self._get_type_and_name_from_technical_name()[0]
+        return self.metadata["connectorType"]
+
+    @property
+    def is_third_party(self) -> bool:
+        return THIRD_PARTY_GLOB in self.relative_connector_path
+
+    @property
+    def has_airbyte_docs(self) -> bool:
+        return (
+            self.metadata
+            and self.metadata.get("documentationUrl") is not None
+            and BASE_AIRBYTE_DOCS_URL in self.metadata.get("documentationUrl")
+        )
 
     @property
     def documentation_directory(self) -> Path:
-        return Path(f"./docs/integrations/{self.connector_type}s")
+        if not self.has_airbyte_docs:
+            return None
+        return get_airbyte_repo_path_with_fallback() / Path(f"docs/integrations/{self.connector_type}s")
 
     @property
-    def documentation_file_path(self) -> Path:
-        readme_file_name = f"{self.name}.md"
-        return self.documentation_directory / readme_file_name
+    def has_airbyte_docs(self) -> bool:
+        docs_url = self.metadata.get("documentationUrl")
+        return docs_url is not None and docs_url.startswith(BASE_AIRBYTE_DOCS_URL)
 
     @property
-    def inapp_documentation_file_path(self) -> Path:
-        readme_file_name = f"{self.name}.inapp.md"
-        return self.documentation_directory / readme_file_name
+    def relative_documentation_path_str(self) -> Optional[str]:
+        if not self.has_airbyte_docs:
+            return None
+
+        documentation_url = self.metadata["documentationUrl"]
+        relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "")
+
+        # strip leading and trailing slashes
+        relative_documentation_path = relative_documentation_path.strip("/")
+
+        return f"docs/{relative_documentation_path}"
 
     @property
-    def migration_guide_file_name(self) -> str:
-        return f"{self.name}-migrations.md"
+    def documentation_file_path(self) -> Optional[Path]:
+        if not self.has_airbyte_docs:
+            return None
+
+        return get_airbyte_repo_path_with_fallback() / Path(f"{self.relative_documentation_path_str}.md")
 
     @property
-    def migration_guide_file_path(self) -> Path:
-        return self.documentation_directory / self.migration_guide_file_name
+    def inapp_documentation_file_path(self) -> Optional[Path]:
+        if not self.has_airbyte_docs:
+            return None
+
+        return get_airbyte_repo_path_with_fallback() / Path(f"{self.relative_documentation_path_str}.inapp.md")
+
+    @property
+    def migration_guide_file_path(self) -> Optional[Path]:
+        if not self.has_airbyte_docs:
+            return None
+        return get_airbyte_repo_path_with_fallback() / Path(f"{self.relative_documentation_path_str}-migrations.md")
 
     @property
     def icon_path(self) -> Path:
@@ -297,7 +372,7 @@ def icon_path(self) -> Path:
 
     @property
     def code_directory(self) -> Path:
-        return Path(f"./airbyte-integrations/connectors/{self.technical_name}")
+        return get_airbyte_repo_path_with_fallback() / Path(f"{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}")
 
     @property
     def has_dockerfile(self) -> bool:
@@ -517,8 +592,7 @@ def get_changed_connectors(
 ) -> Set[Connector]:
     """Retrieve a set of Connectors that were changed in the current branch (compared to master)."""
     if modified_files is None:
-        airbyte_repo = git.Repo(search_parent_directories=True)
-        modified_files = airbyte_repo.git.diff("--name-only", DIFFED_BRANCH).split("\n")
+        modified_files = get_airbyte_repo().git.diff("--name-only", DIFFED_BRANCH).split("\n")
 
     prefix_to_check = []
     if source:
@@ -536,6 +610,26 @@ def get_changed_connectors(
     return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files}
 
 
+def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str:
+    """Get the relative connector folder name from the metadata file path.
+
+    Args:
+        metadata_file_path (Path): Path to the metadata file.
+
+    Returns:
+        str: The relative connector folder name.
+    """
+    # remove CONNECTOR_PATH_PREFIX and anything before
+    metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1]
+
+    # remove metadata.yaml
+    metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "")
+
+    # remove leading and trailing slashes
+    metadata_file_path = metadata_file_path.strip("/")
+    return metadata_file_path
+
+
 def get_all_connectors_in_repo() -> Set[Connector]:
     """Retrieve a set of all Connectors in the repo.
     We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name.
@@ -547,11 +641,9 @@ def get_all_connectors_in_repo() -> Set[Connector]:
     repo_path = repo.working_tree_dir
 
     return {
-        Connector(Path(metadata_file).parent.name)
-        for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
-        # HACK: The Connector util is not good at fetching metadata for third party connectors.
-        # We want to avoid picking a connector that does not have metadata.
-        if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file
+        Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file))
+        for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True)
+        if SCAFFOLD_CONNECTOR_GLOB not in metadata_file
     }
 
 

airbyte-ci/connectors/connector_ops/tests/test_utils.py

@@ -176,5 +176,8 @@ def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies):
 def test_get_all_connectors_in_repo():
     all_connectors = utils.get_all_connectors_in_repo()
     assert len(all_connectors) > 0
-    assert all([isinstance(connector, utils.Connector) for connector in all_connectors])
-    assert all([connector.metadata is not None for connector in all_connectors])
+    for connector in all_connectors:
+        assert isinstance(connector, utils.Connector)
+        assert connector.metadata is not None
+        if connector.has_airbyte_docs:
+            assert connector.documentation_file_path.exists()

airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py

@@ -103,7 +103,8 @@ def get_selected_connectors_with_modified_files(
     selected_connectors_with_modified_files = []
     for connector in selected_connectors:
         connector_with_modified_files = ConnectorWithModifiedFiles(
-            technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files)
+            relative_connector_path=connector.relative_connector_path,
+            modified_files=get_connector_modified_files(connector, modified_files),
         )
         if not metadata_changes_only:
             selected_connectors_with_modified_files.append(connector_with_modified_files)

airbyte-ci/connectors/pipelines/pipelines/tests/common.py

@@ -13,7 +13,7 @@
 import requests
 import semver
 import yaml
-from connector_ops.utils import Connector
+from connector_ops.utils import Connector, abs_project_path_to_relative_path_str
 from dagger import Container, Directory, File
 from pipelines import hacks
 from pipelines.actions import environments
@@ -145,21 +145,21 @@ async def _run(self) -> StepResult:
         """
         connector_ops = await environments.with_connector_ops(self.context)
         include = [
-            str(self.context.connector.code_directory),
-            str(self.context.connector.documentation_file_path),
-            str(self.context.connector.migration_guide_file_path),
-            str(self.context.connector.icon_path),
+            abs_project_path_to_relative_path_str(self.context.connector.code_directory),
+            abs_project_path_to_relative_path_str(self.context.connector.documentation_file_path),
+            abs_project_path_to_relative_path_str(self.context.connector.migration_guide_file_path),
+            abs_project_path_to_relative_path_str(self.context.connector.icon_path),
         ]
         if (
             self.context.connector.technical_name.endswith("strict-encrypt")
             or self.context.connector.technical_name == "source-file-secure"
         ):
             original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", ""))
             include += [
-                str(original_connector.code_directory),
-                str(original_connector.documentation_file_path),
-                str(original_connector.icon_path),
-                str(original_connector.migration_guide_file_path),
+                abs_project_path_to_relative_path_str(original_connector.code_directory),
+                abs_project_path_to_relative_path_str(original_connector.documentation_file_path),
+                abs_project_path_to_relative_path_str(original_connector.icon_path),
+                abs_project_path_to_relative_path_str(original_connector.migration_guide_file_path),
             ]
 
         filtered_repo = self.context.get_repo_dir(

airbyte-ci/connectors/pipelines/pipelines/utils.py

@@ -21,7 +21,7 @@
 import asyncer
 import click
 import git
-from connector_ops.utils import get_changed_connectors
+from connector_ops.utils import abs_project_path_to_relative_path_str, get_changed_connectors
 from dagger import Client, Config, Connection, Container, DaggerError, ExecError, File, ImageLayerCompression, QueryError, Secret
 from google.cloud import storage
 from google.oauth2 import service_account
@@ -330,7 +330,8 @@ def _find_modified_connectors(
     modified_connectors = set()
 
     for connector in all_connectors:
-        if Path(file_path).is_relative_to(Path(connector.code_directory)):
+        relative_code_directory = abs_project_path_to_relative_path_str(connector.code_directory)
+        if Path(file_path).is_relative_to(relative_code_directory):
             main_logger.info(f"Adding connector '{connector}' due to connector file modification: {file_path}.")
             modified_connectors.add(connector)
 

docusaurus/redirects.yml

@@ -5,8 +5,8 @@
   to: /operator-guides/upgrading-airbyte
 - from: /catalog
   to: /understanding-airbyte/airbyte-protocol
-- from: /integrations/sources/appstore
-  to: /integrations/sources/appstore-singer
+- from: /integrations/sources/appstore-singer
+  to: /integrations/sources/appstore
 - from:
     - /project-overview/security
     - /operator-guides/securing-airbyte