From 64e783efc72e72f27fd5752acfb6c6c6e0a56fb2 Mon Sep 17 00:00:00 2001 From: Ben Pankow Date: Mon, 29 Jul 2024 15:57:27 -0700 Subject: [PATCH] Update serverless workflow template (#187) * Update serverless workflow template * rename * respond to review comment --- github/serverless/branch_deployments.yml | 78 --------------- github/serverless/dagster-plus-deploy.yml | 115 ++++++++++++++++++++++ github/serverless/deploy.yml | 80 --------------- 3 files changed, 115 insertions(+), 158 deletions(-) delete mode 100644 github/serverless/branch_deployments.yml create mode 100644 github/serverless/dagster-plus-deploy.yml delete mode 100644 github/serverless/deploy.yml diff --git a/github/serverless/branch_deployments.yml b/github/serverless/branch_deployments.yml deleted file mode 100644 index 6a0f6dcc..00000000 --- a/github/serverless/branch_deployments.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Serverless Branch Deployments -on: - pull_request: - types: [opened, synchronize, reopened, closed] - -concurrency: - # Cancel in-progress deploys to same branch - group: ${{ github.ref }}/branch_deployments - cancel-in-progress: true -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.8' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-20.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - - steps: - - name: Prerun Checks - id: prerun - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-20.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster Cloud serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs - # env_vars: ${{ toJson(secrets) }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/github/serverless/dagster-plus-deploy.yml b/github/serverless/dagster-plus-deploy.yml new file mode 100644 index 00000000..6c6b8a63 --- /dev/null +++ b/github/serverless/dagster-plus-deploy.yml @@ -0,0 +1,115 @@ +name: Dagster Cloud Serverless Deployment +on: + push: + branches: + - "main" + - "master" + pull_request: + types: [opened, synchronize, reopened, closed] + +concurrency: + # Cancel in-progress deploys to same branch + group: ${{ github.ref }}/deploy + cancel-in-progress: true +env: + DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + ENABLE_FAST_DEPLOYS: 'true' + PYTHON_VERSION: '3.8' + DAGSTER_CLOUD_YAML_PATH: '.' + DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' + DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + dagster_cloud_default_deploy: + name: Dagster Serverless Deploy + runs-on: ubuntu-20.04 + + steps: + - name: Prerun Checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + # Validate dagster_cloud.yaml and the connection to dagster.cloud + - name: Validate configuration + id: ci-validate + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci check --project-dir ${{ env.DAGSTER_CLOUD_YAML_PATH }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}" + + # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session + - name: Initialize build session + id: ci-init + uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1 + with: + project_dir: ${{ env.DAGSTER_CLOUD_YAML_PATH }} + dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }} + # The full deployment name. If this run is for a PR, this value is ignored and a branch + # deployment is used. + deployment: 'prod' + + + # If using fast build, build the PEX + # First ensure the correct Python version is installed + - name: Set up Python ${{ env.PYTHON_VERSION }} for target + id: custom-python-version + if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install setuptools + if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} + run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools + shell: bash + + - name: Run PEX build + id: run-pex-build + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci build --build-strategy=python-executable --python-version ${{ env.PYTHON_VERSION }} --pex-deps-cache-from='${{ github.repository }}' --pex-deps-cache-to='${{ github.repository }}'" + + + # Otherwise, enable buildx for caching and build the Docker image + - name: Set up Docker Buildx + if: steps.prerun.outputs.result == 'docker-deploy' + uses: docker/setup-buildx-action@v2 + + - name: Run Docker build + id: run-docker-build + if: steps.prerun.outputs.result == 'docker-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci build --build-strategy=docker --python-version ${{ env.PYTHON_VERSION }}" + + + # Deploy all code locations in this build session to Dagster Cloud + - name: Deploy to Dagster Cloud + id: ci-deploy + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci deploy" + + # Update a PR comment - this runs always() so the comment is updated on success and failure + - name: Update PR comment for branch deployments + id: ci-notify + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci notify --project-dir=${{ env.DAGSTER_CLOUD_YAML_PATH }}" + + # Generate a summary that shows up on the Workflow Summary page + - name: Generate a summary + id: ci-summary + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY" diff --git a/github/serverless/deploy.yml b/github/serverless/deploy.yml deleted file mode 100644 index ab419167..00000000 --- a/github/serverless/deploy.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Serverless Prod Deployment -on: - push: - branches: - - "main" - - "master" - -concurrency: - # Cancel in-progress deploys to same branch - group: ${{ github.ref }}/deploy - cancel-in-progress: true -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.8' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-20.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - - steps: - - name: Prerun Checks - id: prerun - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-20.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster Cloud serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs - # env_vars: ${{ toJson(secrets) }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -