diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index 0f24ed63ba34..f3336153b295 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -24,6 +24,7 @@ DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" +BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-" DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-" @@ -32,7 +33,6 @@ THIRD_PARTY_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + f"/{THIRD_PARTY_GLOB}/" SCAFFOLD_CONNECTOR_GLOB = "-scaffold-" - ACCEPTANCE_TEST_CONFIG_FILE_NAME = "acceptance-test-config.yml" AIRBYTE_DOCKER_REPO = "airbyte" AIRBYTE_REPO_DIRECTORY_NAME = "airbyte" @@ -66,6 +66,39 @@ def download_catalog(catalog_url): } +def get_airbyte_repo() -> git.Repo: + """Get the airbyte repo.""" + return git.Repo(search_parent_directories=True) + + +@functools.lru_cache(maxsize=1) +def get_airbyte_repo_path_with_fallback() -> Path: + """Get the airbyte repo path.""" + try: + return get_airbyte_repo().working_tree_dir + except git.exc.InvalidGitRepositoryError: + logging.warning("Could not find the airbyte repo, falling back to the current working directory.") + path = Path.cwd() + logging.warning(f"Using {path} as the airbyte repo path.") + # log the files in the current directory to help debug + logging.warning(f"Files in {path}: {os.listdir(path)}") + return path + + +def abs_project_path_to_relative_path_str(absolute_path: Path) -> str: + """Get the relative path from the absolute path. + + e.g. /Users/airbyte/airbyte/integrations/source-google-sheets -> integrations/source-google-sheets + + Args: + absolute_path (Path): The absolute path. + + Returns: + str: The relative path string. + """ + return str(absolute_path).replace(get_airbyte_repo_path_with_fallback(), "").strip("/") + + class ConnectorInvalidNameError(Exception): pass @@ -87,7 +120,6 @@ def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[ Returns: Set[Connector]: Set of connectors that were changed """ - airbyte_repo = git.Repo(search_parent_directories=True) if diff_regex is None: diff_command_args = ("--name-only", DIFFED_BRANCH) @@ -96,7 +128,7 @@ def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[ changed_acceptance_test_config_paths = { file_path - for file_path in airbyte_repo.git.diff(*diff_command_args).split("\n") + for file_path in get_airbyte_repo().git.diff(*diff_command_args).split("\n") if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX) and file_path.endswith(ACCEPTANCE_TEST_CONFIG_FILE_NAME) } return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_acceptance_test_config_paths} @@ -251,7 +283,8 @@ class ConnectorLanguageError(Exception): class Connector: """Utility class to gather metadata about a connector.""" - technical_name: str + # The technical name of the connector, e.g. source-google-sheets or third-party/farosai/airbyte-pagerduty-source + relative_connector_path: str def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: if "-" not in self.technical_name: @@ -260,35 +293,77 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: name = self.technical_name[len(_type) + 1 :] return _type, name + @property + def technical_name(self) -> str: + """ + Return the technical name of the connector from the given relative_connector_path + e.g. source-google-sheets -> source-google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source + """ + return self.relative_connector_path.split("/")[-1] + @property def name(self): return self._get_type_and_name_from_technical_name()[1] @property def connector_type(self) -> str: - return self._get_type_and_name_from_technical_name()[0] + return self.metadata["connectorType"] + + @property + def is_third_party(self) -> bool: + return THIRD_PARTY_GLOB in self.relative_connector_path + + @property + def has_airbyte_docs(self) -> bool: + return ( + self.metadata + and self.metadata.get("documentationUrl") is not None + and BASE_AIRBYTE_DOCS_URL in self.metadata.get("documentationUrl") + ) @property def documentation_directory(self) -> Path: - return Path(f"./docs/integrations/{self.connector_type}s") + if not self.has_airbyte_docs: + return None + return get_airbyte_repo_path_with_fallback() / Path(f"docs/integrations/{self.connector_type}s") @property - def documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.md" - return self.documentation_directory / readme_file_name + def has_airbyte_docs(self) -> bool: + docs_url = self.metadata.get("documentationUrl") + return docs_url is not None and docs_url.startswith(BASE_AIRBYTE_DOCS_URL) @property - def inapp_documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.inapp.md" - return self.documentation_directory / readme_file_name + def relative_documentation_path_str(self) -> Optional[str]: + if not self.has_airbyte_docs: + return None + + documentation_url = self.metadata["documentationUrl"] + relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "") + + # strip leading and trailing slashes + relative_documentation_path = relative_documentation_path.strip("/") + + return f"docs/{relative_documentation_path}" @property - def migration_guide_file_name(self) -> str: - return f"{self.name}-migrations.md" + def documentation_file_path(self) -> Optional[Path]: + if not self.has_airbyte_docs: + return None + + return get_airbyte_repo_path_with_fallback() / Path(f"{self.relative_documentation_path_str}.md") @property - def migration_guide_file_path(self) -> Path: - return self.documentation_directory / self.migration_guide_file_name + def inapp_documentation_file_path(self) -> Optional[Path]: + if not self.has_airbyte_docs: + return None + + return get_airbyte_repo_path_with_fallback() / Path(f"{self.relative_documentation_path_str}.inapp.md") + + @property + def migration_guide_file_path(self) -> Optional[Path]: + if not self.has_airbyte_docs: + return None + return get_airbyte_repo_path_with_fallback() / Path(f"{self.relative_documentation_path_str}-migrations.md") @property def icon_path(self) -> Path: @@ -297,7 +372,7 @@ def icon_path(self) -> Path: @property def code_directory(self) -> Path: - return Path(f"./airbyte-integrations/connectors/{self.technical_name}") + return get_airbyte_repo_path_with_fallback() / Path(f"{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}") @property def has_dockerfile(self) -> bool: @@ -517,8 +592,7 @@ def get_changed_connectors( ) -> Set[Connector]: """Retrieve a set of Connectors that were changed in the current branch (compared to master).""" if modified_files is None: - airbyte_repo = git.Repo(search_parent_directories=True) - modified_files = airbyte_repo.git.diff("--name-only", DIFFED_BRANCH).split("\n") + modified_files = get_airbyte_repo().git.diff("--name-only", DIFFED_BRANCH).split("\n") prefix_to_check = [] if source: @@ -536,6 +610,26 @@ def get_changed_connectors( return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files} +def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str: + """Get the relative connector folder name from the metadata file path. + + Args: + metadata_file_path (Path): Path to the metadata file. + + Returns: + str: The relative connector folder name. + """ + # remove CONNECTOR_PATH_PREFIX and anything before + metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1] + + # remove metadata.yaml + metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "") + + # remove leading and trailing slashes + metadata_file_path = metadata_file_path.strip("/") + return metadata_file_path + + def get_all_connectors_in_repo() -> Set[Connector]: """Retrieve a set of all Connectors in the repo. We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name. @@ -547,11 +641,9 @@ def get_all_connectors_in_repo() -> Set[Connector]: repo_path = repo.working_tree_dir return { - Connector(Path(metadata_file).parent.name) - for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) - # HACK: The Connector util is not good at fetching metadata for third party connectors. - # We want to avoid picking a connector that does not have metadata. - if SCAFFOLD_CONNECTOR_GLOB not in metadata_file and THIRD_PARTY_GLOB not in metadata_file + Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file)) + for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True) + if SCAFFOLD_CONNECTOR_GLOB not in metadata_file } diff --git a/airbyte-ci/connectors/connector_ops/tests/test_utils.py b/airbyte-ci/connectors/connector_ops/tests/test_utils.py index f8d61cd3ba2e..fe74983ef010 100644 --- a/airbyte-ci/connectors/connector_ops/tests/test_utils.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_utils.py @@ -176,5 +176,8 @@ def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies): def test_get_all_connectors_in_repo(): all_connectors = utils.get_all_connectors_in_repo() assert len(all_connectors) > 0 - assert all([isinstance(connector, utils.Connector) for connector in all_connectors]) - assert all([connector.metadata is not None for connector in all_connectors]) + for connector in all_connectors: + assert isinstance(connector, utils.Connector) + assert connector.metadata is not None + if connector.has_airbyte_docs: + assert connector.documentation_file_path.exists() diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py index b4b8e0d7711e..30d06cd20ca0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py @@ -103,7 +103,8 @@ def get_selected_connectors_with_modified_files( selected_connectors_with_modified_files = [] for connector in selected_connectors: connector_with_modified_files = ConnectorWithModifiedFiles( - technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files) + relative_connector_path=connector.relative_connector_path, + modified_files=get_connector_modified_files(connector, modified_files), ) if not metadata_changes_only: selected_connectors_with_modified_files.append(connector_with_modified_files) diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/common.py b/airbyte-ci/connectors/pipelines/pipelines/tests/common.py index 253a9257434c..36dcb519786d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/common.py @@ -13,7 +13,7 @@ import requests import semver import yaml -from connector_ops.utils import Connector +from connector_ops.utils import Connector, abs_project_path_to_relative_path_str from dagger import Container, Directory, File from pipelines import hacks from pipelines.actions import environments @@ -145,10 +145,10 @@ async def _run(self) -> StepResult: """ connector_ops = await environments.with_connector_ops(self.context) include = [ - str(self.context.connector.code_directory), - str(self.context.connector.documentation_file_path), - str(self.context.connector.migration_guide_file_path), - str(self.context.connector.icon_path), + abs_project_path_to_relative_path_str(self.context.connector.code_directory), + abs_project_path_to_relative_path_str(self.context.connector.documentation_file_path), + abs_project_path_to_relative_path_str(self.context.connector.migration_guide_file_path), + abs_project_path_to_relative_path_str(self.context.connector.icon_path), ] if ( self.context.connector.technical_name.endswith("strict-encrypt") @@ -156,10 +156,10 @@ async def _run(self) -> StepResult: ): original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", "")) include += [ - str(original_connector.code_directory), - str(original_connector.documentation_file_path), - str(original_connector.icon_path), - str(original_connector.migration_guide_file_path), + abs_project_path_to_relative_path_str(original_connector.code_directory), + abs_project_path_to_relative_path_str(original_connector.documentation_file_path), + abs_project_path_to_relative_path_str(original_connector.icon_path), + abs_project_path_to_relative_path_str(original_connector.migration_guide_file_path), ] filtered_repo = self.context.get_repo_dir( diff --git a/airbyte-ci/connectors/pipelines/pipelines/utils.py b/airbyte-ci/connectors/pipelines/pipelines/utils.py index 80396266c73b..694ca696a22f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/utils.py @@ -21,7 +21,7 @@ import asyncer import click import git -from connector_ops.utils import get_changed_connectors +from connector_ops.utils import abs_project_path_to_relative_path_str, get_changed_connectors from dagger import Client, Config, Connection, Container, DaggerError, ExecError, File, ImageLayerCompression, QueryError, Secret from google.cloud import storage from google.oauth2 import service_account @@ -330,7 +330,8 @@ def _find_modified_connectors( modified_connectors = set() for connector in all_connectors: - if Path(file_path).is_relative_to(Path(connector.code_directory)): + relative_code_directory = abs_project_path_to_relative_path_str(connector.code_directory) + if Path(file_path).is_relative_to(relative_code_directory): main_logger.info(f"Adding connector '{connector}' due to connector file modification: {file_path}.") modified_connectors.add(connector) diff --git a/docs/integrations/sources/appstore-singer.md b/docs/integrations/sources/appstore.md similarity index 100% rename from docs/integrations/sources/appstore-singer.md rename to docs/integrations/sources/appstore.md diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml index 1014d9a7758c..b69386db8c1d 100644 --- a/docusaurus/redirects.yml +++ b/docusaurus/redirects.yml @@ -5,8 +5,8 @@ to: /operator-guides/upgrading-airbyte - from: /catalog to: /understanding-airbyte/airbyte-protocol -- from: /integrations/sources/appstore - to: /integrations/sources/appstore-singer +- from: /integrations/sources/appstore-singer + to: /integrations/sources/appstore - from: - /project-overview/security - /operator-guides/securing-airbyte