feat(registry): Add dev deployment

airbytehq · May 15, 2024 · fab294d · fab294d
1 parent 92c0f72
commit fab294d
Show file tree

Hide file tree

Showing 8 changed files with 186 additions and 786 deletions.
diff --git a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml
@@ -14,8 +14,9 @@ jobs:
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
-      - name: Deploy the metadata orchestrator
-        id: metadata-orchestrator-deploy-orchestrator-pipeline
+      - name: Deploy the metadata orchestrator [On merge to master]
+        id: metadata-orchestrator-deploy-orchestrator-pipeline-prod
+        if: github.event_name == 'push' || github.ref == 'refs/heads/master'
         uses: ./.github/actions/run-airbyte-ci
         with:
           subcommand: "metadata deploy orchestrator"
@@ -27,3 +28,19 @@ jobs:
           gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
         env:
           DAGSTER_CLOUD_METADATA_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_METADATA_API_TOKEN }}
+          DAGSTER_CLOUD_DEPLOYMENT: "prod"
+      - name: Deploy the metadata orchestrator [On workflow]
+        id: metadata-orchestrator-deploy-orchestrator-pipeline-branch
+        if: github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/master'
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          subcommand: "metadata deploy orchestrator"
+          context: "manual"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+        env:
+          DAGSTER_CLOUD_METADATA_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_DEV_METADATA_API_TOKEN }}
+          DAGSTER_CLOUD_DEPLOYMENT: "dev"
diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock b/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock
diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml b/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml
@@ -20,9 +20,8 @@ deepdiff = "^6.3.0"
 mergedeep = "^1.3.4"
 pydash = "^6.0.2"
 dpath = "^2.1.5"
+dagster-cloud = "^1.7.5"
 grpcio = "^1.47.0"
-poetry2setup = "^1.1.0"
-poetry = "^1.5.1"
 pydantic = "^1.10.8"
 dagster-slack = "^0.23.5"
 sentry-sdk = "^1.28.1"

diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md
@@ -745,6 +745,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.13.1  | [#32715](https://github.com/airbytehq/airbyte/pull/32715)  | Add dagster cloud dev deployment pipeline opitions                                                                           |
 | 4.13.0  | [#32715](https://github.com/airbytehq/airbyte/pull/32715)  | Tag connector metadata with git info                                                                                       |
 | 4.12.7  | [#37787](https://github.com/airbytehq/airbyte/pull/37787)  | Remove requirements on dockerhub credentials to run QA checks.                                                               |
 | 4.12.6  | [#36497](https://github.com/airbytehq/airbyte/pull/36497)  | Add airbyte-cdk to list of poetry packages for testing                                                                       |

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py
@@ -28,7 +28,7 @@ async def deploy_orchestrator(ctx: click.Context) -> None:
     # Import locally to speed up CLI.
     from pipelines.airbyte_ci.metadata.pipeline import run_metadata_orchestrator_deploy_pipeline
 
-    await run_metadata_orchestrator_deploy_pipeline(
+    return await run_metadata_orchestrator_deploy_pipeline(
         ctx,
         ctx.obj["is_local"],
         ctx.obj["git_branch"],

diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+import os
 import uuid
 from typing import Optional
 
@@ -115,8 +116,6 @@ class DeployOrchestrator(Step):
         "dagster_cloud.yaml",
         "--organization",
         "airbyte-connectors",
-        "--deployment",
-        "prod",
         "--python-version",
         "3.9",
     ]
@@ -125,16 +124,23 @@ async def _run(self) -> StepResult:
         # mount metadata_service/lib and metadata_service/orchestrator
         parent_dir = self.context.get_repo_dir("airbyte-ci/connectors/metadata_service")
         python_base = with_python_base(self.context, "3.9")
-        python_with_dependencies = with_pip_packages(python_base, ["'dagster-cloud[serverless]'==1.7.5", "poetry2setup==1.1.0"])
+        python_with_dependencies = with_pip_packages(python_base, ["dagster-cloud[serverless]==1.7.5", "poetry2setup==1.1.0"])
         dagster_cloud_api_token_secret: dagger.Secret = get_secret_host_variable(
             self.context.dagger_client, "DAGSTER_CLOUD_METADATA_API_TOKEN"
         )
 
+        # get env var DAGSTER_CLOUD_DEPLOYMENT default to dev
+        target_deployment = os.getenv("DAGSTER_CLOUD_DEPLOYMENT", "dev")
+
+        self.context.logger.info(f"Deploying to deployment: {target_deployment}")
+
         container_to_run = (
             python_with_dependencies.with_mounted_directory("/src", parent_dir)
             .with_secret_variable("DAGSTER_CLOUD_API_TOKEN", dagster_cloud_api_token_secret)
+            .with_env_variable("DAGSTER_CLOUD_DEPLOYMENT", target_deployment)
             .with_workdir("/src/orchestrator")
             .with_exec(["/bin/sh", "-c", "poetry2setup >> setup.py"])
+            .with_exec(["/bin/sh", "-c", "cat setup.py"])
             .with_exec(self.deploy_dagster_command)
         )
         return await self.get_step_result(container_to_run)

diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py
@@ -2,6 +2,8 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
+import uuid
+
 from dagger import CacheSharingMode, CacheVolume, Client, Container
 from pipelines.airbyte_ci.connectors.context import PipelineContext
 from pipelines.consts import (
@@ -33,6 +35,7 @@ def with_python_base(context: PipelineContext, python_version: str = "3.10") ->
 
     base_container = (
         context.dagger_client.container()
+        .with_env_variable("CACHE_BUSTER", str(uuid.uuid4()))
         .from_(f"python:{python_version}-slim")
         .with_mounted_cache("/root/.cache/pip", pip_cache)
         .with_exec(

diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.13.0"
+version = "4.13.1"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]