From c97cdb0368d89445c93487089af71aabcaa386c3 Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Thu, 11 Jul 2024 12:11:03 -0700 Subject: [PATCH] airbyte-ci: add and adopt AIRBYTE_GITHUB_REPO env var (#41642) --- airbyte-ci/connectors/pipelines/README.md | 1 + .../pipelines/airbyte_ci/connectors/publish/context.py | 5 +++-- .../pipelines/airbyte_ci/connectors/reports.py | 5 +++-- .../airbyte_ci/connectors/test/steps/common.py | 10 +++++----- .../connectors/pipelines/pipelines/cli/airbyte_ci.py | 7 ++++--- .../pipelines/pipelines/dagger/containers/git.py | 7 ++++--- .../connectors/pipelines/pipelines/helpers/git.py | 10 ++++++++-- .../connectors/pipelines/pipelines/helpers/github.py | 6 +++++- .../connectors/pipelines/pipelines/helpers/utils.py | 1 - .../pipelines/models/contexts/pipeline_context.py | 6 +++--- airbyte-ci/connectors/pipelines/pyproject.toml | 2 +- 11 files changed, 37 insertions(+), 23 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index e1006331a8f2..17113811051b 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -767,6 +767,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| 4.24.1 | [#41642](https://github.com/airbytehq/airbyte/pull/41642) | Use the AIRBYTE_GITHUB_REPO environment variable to run airbyte-ci in other repos. | | 4.24.0 | [#41627](https://github.com/airbytehq/airbyte/pull/41627) | Require manual regression test approval for certified connectors | | 4.23.1 | [#41541](https://github.com/airbytehq/airbyte/pull/41541) | Add support for submodule use-case. | | 4.23.0 | [#39906](https://github.com/airbytehq/airbyte/pull/39906) | Add manifest only build pipeline | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index 250d4c51ba20..b4fda32ca9cc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -11,6 +11,7 @@ from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.consts import ContextState from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles +from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL_PREFIX from pipelines.helpers.utils import format_duration from pipelines.models.secrets import Secret @@ -117,9 +118,9 @@ def create_slack_message(self) -> str: message += "🧑‍💻 Local run\n" message += f"*Connector:* {self.connector.technical_name}\n" message += f"*Version:* {self.connector.version}\n" - branch_url = f"https://github.com/airbytehq/airbyte/tree/{self.git_branch}" + branch_url = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/tree/{self.git_branch}" message += f"*Branch:* <{branch_url}|{self.git_branch}>\n" - commit_url = f"https://github.com/airbytehq/airbyte/commit/{self.git_revision}" + commit_url = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/commit/{self.git_revision}" message += f"*Commit:* <{commit_url}|{self.git_revision[:10]}>\n" if self.state in [ContextState.INITIALIZED, ContextState.RUNNING]: message += "🟠" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py index 3888ae8550ad..68f75275ad03 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -13,6 +13,7 @@ from connector_ops.utils import console # type: ignore from jinja2 import Environment, PackageLoader, select_autoescape from pipelines.consts import GCS_PUBLIC_DOMAIN +from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL_PREFIX, AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX from pipelines.helpers.utils import format_duration from pipelines.models.artifacts import Artifact from pipelines.models.reports import Report @@ -128,13 +129,13 @@ def to_html(self) -> str: } if self.pipeline_context.is_ci: - template_context["commit_url"] = f"https://github.com/airbytehq/airbyte/commit/{self.pipeline_context.git_revision}" + template_context["commit_url"] = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/commit/{self.pipeline_context.git_revision}" template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url template_context[ "icon_url" - ] = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + ] = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" return template.render(template_context) async def save_html_report(self) -> None: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py index ec9cc802bd55..7225663f7652 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -27,6 +27,7 @@ from pipelines.consts import INTERNAL_TOOL_PATHS, CIContext from pipelines.dagger.actions import secrets from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.helpers.github import AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX from pipelines.helpers.utils import METADATA_FILE_NAME, get_exec_result from pipelines.models.artifacts import Artifact from pipelines.models.secrets import Secret @@ -36,12 +37,13 @@ # live_test can't resolve the passed connector container otherwise. from slugify import slugify # type: ignore +GITHUB_URL_PREFIX_FOR_CONNECTORS = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/master/airbyte-integrations/connectors" + class VersionCheck(Step, ABC): """A step to validate the connector version was bumped if files were modified""" context: ConnectorContext - GITHUB_URL_PREFIX_FOR_CONNECTORS = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors" failure_message: ClassVar @property @@ -50,7 +52,7 @@ def should_run(self) -> bool: @property def github_master_metadata_url(self) -> str: - return f"{self.GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}" + return f"{GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}" @cached_property def master_metadata(self) -> Optional[dict]: @@ -393,9 +395,7 @@ async def get_result_log_on_master(self) -> Artifact: Returns: Artifact: The report log of the acceptance tests run on the released image. """ - raw_master_metadata = requests.get( - f"https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors/{self.context.connector.technical_name}/metadata.yaml" - ) + raw_master_metadata = requests.get(f"{GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/metadata.yaml") master_metadata = yaml.safe_load(raw_master_metadata.text) master_docker_image_tag = master_metadata["data"]["dockerImageTag"] released_image = f'{master_metadata["data"]["dockerRepository"]}:{master_docker_image_tag}' diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index c0d6abf2ed98..fb7222ed1929 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -39,7 +39,8 @@ from pipelines.dagger.actions.connector.hooks import get_dagger_sdk_version from pipelines.helpers import github from pipelines.helpers.git import get_current_git_branch, get_current_git_revision -from pipelines.helpers.utils import AIRBYTE_REPO_URL, get_current_epoch_time +from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, AIRBYTE_GITHUB_REPO_URL_PREFIX +from pipelines.helpers.utils import get_current_epoch_time from pipelines.models.secrets import InMemorySecretStore @@ -64,7 +65,7 @@ def _get_gha_workflow_run_url(ctx: click.Context) -> Optional[str]: if not gha_workflow_run_id: return None - return f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" + return f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/actions/runs/{gha_workflow_run_id}" def _get_pull_request(ctx: click.Context) -> Optional[PullRequest.PullRequest]: @@ -148,7 +149,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: @click.option("--enable-update-check/--disable-update-check", default=True) @click.option("--enable-auto-update/--disable-auto-update", default=True) @click.option("--is-local/--is-ci", default=True) -@click.option("--git-repo-url", default=AIRBYTE_REPO_URL, envvar="CI_GIT_REPO_URL") +@click.option("--git-repo-url", default=AIRBYTE_GITHUB_REPO_URL, envvar="CI_GIT_REPO_URL") @click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH") @click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION") @click.option( diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py index 721ec4186978..053f321e1651 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py @@ -4,7 +4,8 @@ from typing import Optional from dagger import Client, Container -from pipelines.helpers.utils import AIRBYTE_REPO_URL, sh_dash_c +from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL +from pipelines.helpers.utils import sh_dash_c def get_authenticated_repo_url(url: str, github_token: str) -> str: @@ -16,7 +17,7 @@ async def checked_out_git_container( current_git_branch: str, current_git_revision: str, diffed_branch: Optional[str] = None, - repo_url: str = AIRBYTE_REPO_URL, + repo_url: str = AIRBYTE_GITHUB_REPO_URL, ) -> Container: """ Create a container with git in it. @@ -24,7 +25,7 @@ async def checked_out_git_container( We fetch the diffed branch from the origin remote and the current branch from the target remote. We then checkout the current branch. """ - origin_repo_url = AIRBYTE_REPO_URL + origin_repo_url = AIRBYTE_GITHUB_REPO_URL current_git_branch = current_git_branch.removeprefix("origin/") diffed_branch = current_git_branch if diffed_branch is None else diffed_branch.removeprefix("origin/") if github_token := os.environ.get("CI_GITHUB_ACCESS_TOKEN"): diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py index bedbf5c8f7f4..b25c4b1a6595 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py @@ -9,7 +9,8 @@ from dagger import Connection, SessionError from pipelines.consts import CIContext from pipelines.dagger.containers.git import checked_out_git_container -from pipelines.helpers.utils import AIRBYTE_REPO_URL, DAGGER_CONFIG, DIFF_FILTER +from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL +from pipelines.helpers.utils import DAGGER_CONFIG, DIFF_FILTER def get_current_git_revision() -> str: # noqa D103 @@ -103,7 +104,12 @@ def get_git_repo_path() -> str: async def get_modified_files( - git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext, git_repo_url: str = AIRBYTE_REPO_URL + git_branch: str, + git_revision: str, + diffed_branch: str, + is_local: bool, + ci_context: CIContext, + git_repo_url: str = AIRBYTE_GITHUB_REPO_URL, ) -> Set[str]: """Get the list of modified files in the current git branch. If the current branch is master, it will return the list of modified files in the head commit. diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py index abb1c7618931..c7dee7e5e65a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py @@ -23,7 +23,11 @@ from typing import Iterable, List, Optional -AIRBYTE_GITHUB_REPO = "airbytehq/airbyte" +DEFAULT_AIRBYTE_GITHUB_REPO = "airbytehq/airbyte" +AIRBYTE_GITHUB_REPO = os.environ.get("AIRBYTE_GITHUB_REPO", DEFAULT_AIRBYTE_GITHUB_REPO) +AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX = f"https://raw.githubusercontent.com/{AIRBYTE_GITHUB_REPO}" +AIRBYTE_GITHUB_REPO_URL_PREFIX = f"https://github.com/{AIRBYTE_GITHUB_REPO}" +AIRBYTE_GITHUB_REPO_URL = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}.git" BASE_BRANCH = "master" diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py index 5cdd623a80af..13ca8cfa111e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -28,7 +28,6 @@ from pipelines.airbyte_ci.connectors.context import ConnectorContext DAGGER_CONFIG = Config(log_output=sys.stderr) -AIRBYTE_REPO_URL = os.environ.get("AIRBYTE_REPO_URL", "https://github.com/airbytehq/airbyte.git") METADATA_FILE_NAME = "metadata.yaml" MANIFEST_FILE_NAME = "manifest.yaml" METADATA_ICON_FILE_NAME = "icon.svg" diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py index ec9b07e62ec6..924749137e9a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py @@ -22,9 +22,9 @@ from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.consts import CIContext, ContextState from pipelines.helpers.execution.run_steps import RunStepOptions -from pipelines.helpers.github import update_commit_status_check +from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, update_commit_status_check from pipelines.helpers.slack import send_message_to_webhook -from pipelines.helpers.utils import AIRBYTE_REPO_URL, java_log_scrub_pattern +from pipelines.helpers.utils import java_log_scrub_pattern from pipelines.models.reports import Report from pipelines.models.secrets import Secret, SecretStore @@ -159,7 +159,7 @@ def is_pr(self) -> bool: @property def repo(self) -> GitRepository: - return self.dagger_client.git(AIRBYTE_REPO_URL, keep_git_dir=True) + return self.dagger_client.git(AIRBYTE_GITHUB_REPO_URL, keep_git_dir=True) @property def report(self) -> Report | ConnectorReport | None: diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 897b8ac31cc9..e1c3e7e4b5f5 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.24.0" +version = "4.24.1" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "]