airbyte-ci: add and adopt AIRBYTE_GITHUB_REPO env var (#41642)

airbyte-ci/connectors/pipelines/README.md

@@ -767,6 +767,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.24.1  | [#41642](https://github.com/airbytehq/airbyte/pull/41642) | Use the AIRBYTE_GITHUB_REPO environment variable to run airbyte-ci in other repos.                                             |
 | 4.24.0  | [#41627](https://github.com/airbytehq/airbyte/pull/41627) | Require manual regression test approval for certified connectors                                                             |
 | 4.23.1  | [#41541](https://github.com/airbytehq/airbyte/pull/41541)  | Add support for submodule use-case.                                                                                          |
 | 4.23.0  | [#39906](https://github.com/airbytehq/airbyte/pull/39906)  | Add manifest only build pipeline                                                          |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py

@@ -11,6 +11,7 @@
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.consts import ContextState
 from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles
+from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL_PREFIX
 from pipelines.helpers.utils import format_duration
 from pipelines.models.secrets import Secret
 
@@ -117,9 +118,9 @@ def create_slack_message(self) -> str:
             message += "🧑‍💻 Local run\n"
         message += f"*Connector:* {self.connector.technical_name}\n"
         message += f"*Version:* {self.connector.version}\n"
-        branch_url = f"https://github.com/airbytehq/airbyte/tree/{self.git_branch}"
+        branch_url = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/tree/{self.git_branch}"
         message += f"*Branch:* <{branch_url}|{self.git_branch}>\n"
-        commit_url = f"https://github.com/airbytehq/airbyte/commit/{self.git_revision}"
+        commit_url = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/commit/{self.git_revision}"
         message += f"*Commit:* <{commit_url}|{self.git_revision[:10]}>\n"
         if self.state in [ContextState.INITIALIZED, ContextState.RUNNING]:
             message += "🟠"

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py

@@ -13,6 +13,7 @@
 from connector_ops.utils import console  # type: ignore
 from jinja2 import Environment, PackageLoader, select_autoescape
 from pipelines.consts import GCS_PUBLIC_DOMAIN
+from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL_PREFIX, AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX
 from pipelines.helpers.utils import format_duration
 from pipelines.models.artifacts import Artifact
 from pipelines.models.reports import Report
@@ -128,13 +129,13 @@ def to_html(self) -> str:
         }
 
         if self.pipeline_context.is_ci:
-            template_context["commit_url"] = f"https://github.com/airbytehq/airbyte/commit/{self.pipeline_context.git_revision}"
+            template_context["commit_url"] = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/commit/{self.pipeline_context.git_revision}"
             template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url
             template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url
             template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url
             template_context[
                 "icon_url"
-            ] = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg"
+            ] = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg"
         return template.render(template_context)
 
     async def save_html_report(self) -> None:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py

@@ -27,6 +27,7 @@
 from pipelines.consts import INTERNAL_TOOL_PATHS, CIContext
 from pipelines.dagger.actions import secrets
 from pipelines.dagger.actions.python.poetry import with_poetry
+from pipelines.helpers.github import AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX
 from pipelines.helpers.utils import METADATA_FILE_NAME, get_exec_result
 from pipelines.models.artifacts import Artifact
 from pipelines.models.secrets import Secret
@@ -36,12 +37,13 @@
 # live_test can't resolve the passed connector container otherwise.
 from slugify import slugify  # type: ignore
 
+GITHUB_URL_PREFIX_FOR_CONNECTORS = f"{AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX}/master/airbyte-integrations/connectors"
+
 
 class VersionCheck(Step, ABC):
     """A step to validate the connector version was bumped if files were modified"""
 
     context: ConnectorContext
-    GITHUB_URL_PREFIX_FOR_CONNECTORS = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors"
     failure_message: ClassVar
 
     @property
@@ -50,7 +52,7 @@ def should_run(self) -> bool:
 
     @property
     def github_master_metadata_url(self) -> str:
-        return f"{self.GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}"
+        return f"{GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}"
 
     @cached_property
     def master_metadata(self) -> Optional[dict]:
@@ -393,9 +395,7 @@ async def get_result_log_on_master(self) -> Artifact:
         Returns:
             Artifact: The report log of the acceptance tests run on the released image.
         """
-        raw_master_metadata = requests.get(
-            f"https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors/{self.context.connector.technical_name}/metadata.yaml"
-        )
+        raw_master_metadata = requests.get(f"{GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/metadata.yaml")
         master_metadata = yaml.safe_load(raw_master_metadata.text)
         master_docker_image_tag = master_metadata["data"]["dockerImageTag"]
         released_image = f'{master_metadata["data"]["dockerRepository"]}:{master_docker_image_tag}'

airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py

@@ -39,7 +39,8 @@
 from pipelines.dagger.actions.connector.hooks import get_dagger_sdk_version
 from pipelines.helpers import github
 from pipelines.helpers.git import get_current_git_branch, get_current_git_revision
-from pipelines.helpers.utils import AIRBYTE_REPO_URL, get_current_epoch_time
+from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, AIRBYTE_GITHUB_REPO_URL_PREFIX
+from pipelines.helpers.utils import get_current_epoch_time
 from pipelines.models.secrets import InMemorySecretStore
 
 
@@ -64,7 +65,7 @@ def _get_gha_workflow_run_url(ctx: click.Context) -> Optional[str]:
     if not gha_workflow_run_id:
         return None
 
-    return f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}"
+    return f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}/actions/runs/{gha_workflow_run_id}"
 
 
 def _get_pull_request(ctx: click.Context) -> Optional[PullRequest.PullRequest]:
@@ -148,7 +149,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool:
 @click.option("--enable-update-check/--disable-update-check", default=True)
 @click.option("--enable-auto-update/--disable-auto-update", default=True)
 @click.option("--is-local/--is-ci", default=True)
-@click.option("--git-repo-url", default=AIRBYTE_REPO_URL, envvar="CI_GIT_REPO_URL")
+@click.option("--git-repo-url", default=AIRBYTE_GITHUB_REPO_URL, envvar="CI_GIT_REPO_URL")
 @click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH")
 @click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION")
 @click.option(

airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py

@@ -4,7 +4,8 @@
 from typing import Optional
 
 from dagger import Client, Container
-from pipelines.helpers.utils import AIRBYTE_REPO_URL, sh_dash_c
+from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL
+from pipelines.helpers.utils import sh_dash_c
 
 
 def get_authenticated_repo_url(url: str, github_token: str) -> str:
@@ -16,15 +17,15 @@ async def checked_out_git_container(
     current_git_branch: str,
     current_git_revision: str,
     diffed_branch: Optional[str] = None,
-    repo_url: str = AIRBYTE_REPO_URL,
+    repo_url: str = AIRBYTE_GITHUB_REPO_URL,
 ) -> Container:
     """
     Create a container with git in it.
     We add the airbyte repo as the origin remote and the target repo as the target remote.
     We fetch the diffed branch from the origin remote and the current branch from the target remote.
     We then checkout the current branch.
     """
-    origin_repo_url = AIRBYTE_REPO_URL
+    origin_repo_url = AIRBYTE_GITHUB_REPO_URL
     current_git_branch = current_git_branch.removeprefix("origin/")
     diffed_branch = current_git_branch if diffed_branch is None else diffed_branch.removeprefix("origin/")
     if github_token := os.environ.get("CI_GITHUB_ACCESS_TOKEN"):

airbyte-ci/connectors/pipelines/pipelines/helpers/git.py

@@ -9,7 +9,8 @@
 from dagger import Connection, SessionError
 from pipelines.consts import CIContext
 from pipelines.dagger.containers.git import checked_out_git_container
-from pipelines.helpers.utils import AIRBYTE_REPO_URL, DAGGER_CONFIG, DIFF_FILTER
+from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL
+from pipelines.helpers.utils import DAGGER_CONFIG, DIFF_FILTER
 
 
 def get_current_git_revision() -> str:  # noqa D103
@@ -103,7 +104,12 @@ def get_git_repo_path() -> str:
 
 
 async def get_modified_files(
-    git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext, git_repo_url: str = AIRBYTE_REPO_URL
+    git_branch: str,
+    git_revision: str,
+    diffed_branch: str,
+    is_local: bool,
+    ci_context: CIContext,
+    git_repo_url: str = AIRBYTE_GITHUB_REPO_URL,
 ) -> Set[str]:
     """Get the list of modified files in the current git branch.
     If the current branch is master, it will return the list of modified files in the head commit.

airbyte-ci/connectors/pipelines/pipelines/helpers/github.py

@@ -23,7 +23,11 @@
     from typing import Iterable, List, Optional
 
 
-AIRBYTE_GITHUB_REPO = "airbytehq/airbyte"
+DEFAULT_AIRBYTE_GITHUB_REPO = "airbytehq/airbyte"
+AIRBYTE_GITHUB_REPO = os.environ.get("AIRBYTE_GITHUB_REPO", DEFAULT_AIRBYTE_GITHUB_REPO)
+AIRBYTE_GITHUBUSERCONTENT_URL_PREFIX = f"https://raw.githubusercontent.com/{AIRBYTE_GITHUB_REPO}"
+AIRBYTE_GITHUB_REPO_URL_PREFIX = f"https://github.com/{AIRBYTE_GITHUB_REPO}"
+AIRBYTE_GITHUB_REPO_URL = f"{AIRBYTE_GITHUB_REPO_URL_PREFIX}.git"
 BASE_BRANCH = "master"
 
 

airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py

@@ -28,7 +28,6 @@
     from pipelines.airbyte_ci.connectors.context import ConnectorContext
 
 DAGGER_CONFIG = Config(log_output=sys.stderr)
-AIRBYTE_REPO_URL = os.environ.get("AIRBYTE_REPO_URL", "https://github.com/airbytehq/airbyte.git")
 METADATA_FILE_NAME = "metadata.yaml"
 MANIFEST_FILE_NAME = "manifest.yaml"
 METADATA_ICON_FILE_NAME = "icon.svg"

airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py

@@ -22,9 +22,9 @@
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport
 from pipelines.consts import CIContext, ContextState
 from pipelines.helpers.execution.run_steps import RunStepOptions
-from pipelines.helpers.github import update_commit_status_check
+from pipelines.helpers.github import AIRBYTE_GITHUB_REPO_URL, update_commit_status_check
 from pipelines.helpers.slack import send_message_to_webhook
-from pipelines.helpers.utils import AIRBYTE_REPO_URL, java_log_scrub_pattern
+from pipelines.helpers.utils import java_log_scrub_pattern
 from pipelines.models.reports import Report
 from pipelines.models.secrets import Secret, SecretStore
 
@@ -159,7 +159,7 @@ def is_pr(self) -> bool:
 
     @property
     def repo(self) -> GitRepository:
-        return self.dagger_client.git(AIRBYTE_REPO_URL, keep_git_dir=True)
+        return self.dagger_client.git(AIRBYTE_GITHUB_REPO_URL, keep_git_dir=True)
 
     @property
     def report(self) -> Report | ConnectorReport | None:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.24.0"
+version = "4.24.1"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]