From 084f709d7558da308196289f5da2d5b5867f52dc Mon Sep 17 00:00:00 2001 From: gibsondan Date: Tue, 3 Sep 2024 15:33:55 -0500 Subject: [PATCH] Revert "Update serverless workflow template (#187)" This reverts commit 64e783efc72e72f27fd5752acfb6c6c6e0a56fb2. --- github/serverless/branch_deployments.yml | 78 +++++++++++++++ github/serverless/dagster-plus-deploy.yml | 115 ---------------------- github/serverless/deploy.yml | 80 +++++++++++++++ 3 files changed, 158 insertions(+), 115 deletions(-) create mode 100644 github/serverless/branch_deployments.yml delete mode 100644 github/serverless/dagster-plus-deploy.yml create mode 100644 github/serverless/deploy.yml diff --git a/github/serverless/branch_deployments.yml b/github/serverless/branch_deployments.yml new file mode 100644 index 00000000..6a0f6dcc --- /dev/null +++ b/github/serverless/branch_deployments.yml @@ -0,0 +1,78 @@ +name: Serverless Branch Deployments +on: + pull_request: + types: [opened, synchronize, reopened, closed] + +concurrency: + # Cancel in-progress deploys to same branch + group: ${{ github.ref }}/branch_deployments + cancel-in-progress: true +env: + DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + ENABLE_FAST_DEPLOYS: 'true' + PYTHON_VERSION: '3.8' + DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' + +jobs: + dagster_cloud_default_deploy: + name: Dagster Serverless Deploy + runs-on: ubuntu-20.04 + outputs: + build_info: ${{ steps.parse-workspace.outputs.build_info }} + + steps: + - name: Prerun Checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 + + - name: Launch Docker Deploy + if: steps.prerun.outputs.result == 'docker-deploy' + id: parse-workspace + uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 + with: + dagster_cloud_file: $DAGSTER_CLOUD_FILE + + - name: Checkout for Python Executable Deploy + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + path: project-repo + + - name: Python Executable Deploy + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: "${{ env.PYTHON_VERSION }}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + dagster_cloud_docker_deploy: + name: Docker Deploy + runs-on: ubuntu-20.04 + if: needs.dagster_cloud_default_deploy.outputs.build_info + needs: dagster_cloud_default_deploy + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + - name: Build and deploy to Dagster Cloud serverless + uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1 + with: + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location: ${{ toJson(matrix.location) }} + base_image: "python:${{ env.PYTHON_VERSION }}-slim" + # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs + # env_vars: ${{ toJson(secrets) }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/github/serverless/dagster-plus-deploy.yml b/github/serverless/dagster-plus-deploy.yml deleted file mode 100644 index 6c6b8a63..00000000 --- a/github/serverless/dagster-plus-deploy.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Dagster Cloud Serverless Deployment -on: - push: - branches: - - "main" - - "master" - pull_request: - types: [opened, synchronize, reopened, closed] - -concurrency: - # Cancel in-progress deploys to same branch - group: ${{ github.ref }}/deploy - cancel-in-progress: true -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.8' - DAGSTER_CLOUD_YAML_PATH: '.' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-20.04 - - steps: - - name: Prerun Checks - id: prerun - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - # Validate dagster_cloud.yaml and the connection to dagster.cloud - - name: Validate configuration - id: ci-validate - if: steps.prerun.outputs.result != 'skip' - uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 - with: - command: "ci check --project-dir ${{ env.DAGSTER_CLOUD_YAML_PATH }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}" - - # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session - - name: Initialize build session - id: ci-init - uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1 - with: - project_dir: ${{ env.DAGSTER_CLOUD_YAML_PATH }} - dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }} - # The full deployment name. If this run is for a PR, this value is ignored and a branch - # deployment is used. - deployment: 'prod' - - - # If using fast build, build the PEX - # First ensure the correct Python version is installed - - name: Set up Python ${{ env.PYTHON_VERSION }} for target - id: custom-python-version - if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install setuptools - if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} - run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools - shell: bash - - - name: Run PEX build - id: run-pex-build - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 - with: - command: "ci build --build-strategy=python-executable --python-version ${{ env.PYTHON_VERSION }} --pex-deps-cache-from='${{ github.repository }}' --pex-deps-cache-to='${{ github.repository }}'" - - - # Otherwise, enable buildx for caching and build the Docker image - - name: Set up Docker Buildx - if: steps.prerun.outputs.result == 'docker-deploy' - uses: docker/setup-buildx-action@v2 - - - name: Run Docker build - id: run-docker-build - if: steps.prerun.outputs.result == 'docker-deploy' - uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 - with: - command: "ci build --build-strategy=docker --python-version ${{ env.PYTHON_VERSION }}" - - - # Deploy all code locations in this build session to Dagster Cloud - - name: Deploy to Dagster Cloud - id: ci-deploy - if: steps.prerun.outputs.result != 'skip' - uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 - with: - command: "ci deploy" - - # Update a PR comment - this runs always() so the comment is updated on success and failure - - name: Update PR comment for branch deployments - id: ci-notify - if: steps.prerun.outputs.result != 'skip' - uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 - with: - command: "ci notify --project-dir=${{ env.DAGSTER_CLOUD_YAML_PATH }}" - - # Generate a summary that shows up on the Workflow Summary page - - name: Generate a summary - id: ci-summary - if: steps.prerun.outputs.result != 'skip' - uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 - with: - command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY" diff --git a/github/serverless/deploy.yml b/github/serverless/deploy.yml new file mode 100644 index 00000000..ab419167 --- /dev/null +++ b/github/serverless/deploy.yml @@ -0,0 +1,80 @@ +name: Serverless Prod Deployment +on: + push: + branches: + - "main" + - "master" + +concurrency: + # Cancel in-progress deploys to same branch + group: ${{ github.ref }}/deploy + cancel-in-progress: true +env: + DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }} + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + ENABLE_FAST_DEPLOYS: 'true' + PYTHON_VERSION: '3.8' + DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' + +jobs: + dagster_cloud_default_deploy: + name: Dagster Serverless Deploy + runs-on: ubuntu-20.04 + outputs: + build_info: ${{ steps.parse-workspace.outputs.build_info }} + + steps: + - name: Prerun Checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 + + - name: Launch Docker Deploy + if: steps.prerun.outputs.result == 'docker-deploy' + id: parse-workspace + uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 + with: + dagster_cloud_file: $DAGSTER_CLOUD_FILE + + - name: Checkout for Python Executable Deploy + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + path: project-repo + + - name: Python Executable Deploy + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 + with: + dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" + build_output_dir: "$GITHUB_WORKSPACE/build" + python_version: "${{ env.PYTHON_VERSION }}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + dagster_cloud_docker_deploy: + name: Docker Deploy + runs-on: ubuntu-20.04 + if: needs.dagster_cloud_default_deploy.outputs.build_info + needs: dagster_cloud_default_deploy + strategy: + fail-fast: false + matrix: + location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + - name: Build and deploy to Dagster Cloud serverless + uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1 + with: + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location: ${{ toJson(matrix.location) }} + base_image: "python:${{ env.PYTHON_VERSION }}-slim" + # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs + # env_vars: ${{ toJson(secrets) }} + organization_id: ${{ secrets.ORGANIZATION_ID }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +