From d6af248782013641db8d0cdba8e3d18bae4e0b37 Mon Sep 17 00:00:00 2001 From: gibsondan Date: Tue, 29 Oct 2024 17:12:49 -0500 Subject: [PATCH] Change default python version in ubuntu image for newly templated serverless orgs, bring in sync with dagster-cloud-action repo (#33) Summary: Now tht python 3.8 is EOL, change the default python version to 3.10 and the default ubuntu version to one that has 3.10 installed by default instead of 3.8. Also make the quickstart use the same yaml that is templated when you create a new serverless org. --- .github/workflows/branch_deployments.yml | 78 --------------- .github/workflows/dagster-plus-deploy.yml | 114 ++++++++++++++++++++++ .github/workflows/deploy.yml | 80 --------------- README.md | 8 +- dagster_cloud.yaml | 2 +- 5 files changed, 119 insertions(+), 163 deletions(-) delete mode 100644 .github/workflows/branch_deployments.yml create mode 100644 .github/workflows/dagster-plus-deploy.yml delete mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/branch_deployments.yml b/.github/workflows/branch_deployments.yml deleted file mode 100644 index 70b97a5..0000000 --- a/.github/workflows/branch_deployments.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Serverless Branch Deployments -on: - pull_request: - types: [opened, synchronize, reopened, closed] - -concurrency: - # Cancel in-progress deploys to same branch - group: ${{ github.ref }}/branch_deployments - cancel-in-progress: true -env: - DAGSTER_CLOUD_URL: "http://ORGANIZATION_NAME.dagster.cloud" - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.8' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-20.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - - steps: - - name: Prerun Checks - id: prerun - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-20.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster Cloud serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs - # env_vars: ${{ toJson(secrets) }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.github/workflows/dagster-plus-deploy.yml b/.github/workflows/dagster-plus-deploy.yml new file mode 100644 index 0000000..72a4bb9 --- /dev/null +++ b/.github/workflows/dagster-plus-deploy.yml @@ -0,0 +1,114 @@ +name: Dagster Cloud Serverless Deployment +on: + push: + branches: + - "main" + - "master" + pull_request: + types: [opened, synchronize, reopened, closed] + +concurrency: + # Cancel in-progress deploys to same branch + group: ${{ github.ref }}/deploy + cancel-in-progress: true +env: + DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + ENABLE_FAST_DEPLOYS: 'true' + PYTHON_VERSION: '3.10' + DAGSTER_CLOUD_YAML_PATH: '.' + DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' + DAGSTER_CLOUD_ORGANIZATION: 'ORGANIZATION_NAME' + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + dagster_cloud_default_deploy: + name: Dagster Serverless Deploy + runs-on: ubuntu-22.04 + + steps: + - name: Prerun Checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + # Validate dagster_cloud.yaml and the connection to dagster.cloud + - name: Validate configuration + id: ci-validate + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci check --project-dir ${{ env.DAGSTER_CLOUD_YAML_PATH }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}" + + # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session + - name: Initialize build session + id: ci-init + uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1 + with: + project_dir: ${{ env.DAGSTER_CLOUD_YAML_PATH }} + dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }} + # The full deployment name. If this run is for a PR, this value is ignored and a branch + # deployment is used. + deployment: 'prod' + + + # If using fast build, build the PEX + # First ensure the correct Python version is installed + - name: Set up Python ${{ env.PYTHON_VERSION }} for target + id: setup-python-version + if: steps.prerun.outputs.result == 'pex-deploy' + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install setuptools + if: steps.prerun.outputs.result == 'pex-deploy' + run: ${{ steps.setup-python-version.outputs.python-path }} -m pip install setuptools + shell: bash + + - name: Run PEX build + id: run-pex-build + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci build --build-strategy=python-executable --python-version ${{ env.PYTHON_VERSION }} --pex-deps-cache-from='${{ github.repository }}' --pex-deps-cache-to='${{ github.repository }}'" + + + # Otherwise, enable buildx for caching and build the Docker image + - name: Set up Docker Buildx + if: steps.prerun.outputs.result == 'docker-deploy' + uses: docker/setup-buildx-action@v2 + + - name: Run Docker build + id: run-docker-build + if: steps.prerun.outputs.result == 'docker-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci build --build-strategy=docker --python-version ${{ env.PYTHON_VERSION }}" + + + # Deploy all code locations in this build session to Dagster Cloud + - name: Deploy to Dagster Cloud + id: ci-deploy + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci deploy" + + # Update a PR comment - this runs always() so the comment is updated on success and failure + - name: Update PR comment for branch deployments + id: ci-notify + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci notify --project-dir=${{ env.DAGSTER_CLOUD_YAML_PATH }}" + + # Generate a summary that shows up on the Workflow Summary page + - name: Generate a summary + id: ci-summary + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 651d14d..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Serverless Prod Deployment -on: - push: - branches: - - "main" - - "master" - -concurrency: - # Cancel in-progress deploys to same branch - group: ${{ github.ref }}/deploy - cancel-in-progress: true -env: - DAGSTER_CLOUD_URL: "http://ORGANIZATION_NAME.dagster.cloud" - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - ENABLE_FAST_DEPLOYS: 'true' - PYTHON_VERSION: '3.8' - DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' - -jobs: - dagster_cloud_default_deploy: - name: Dagster Serverless Deploy - runs-on: ubuntu-20.04 - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - - steps: - - name: Prerun Checks - id: prerun - uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 - - - name: Launch Docker Deploy - if: steps.prerun.outputs.result == 'docker-deploy' - id: parse-workspace - uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v0.1 - with: - dagster_cloud_file: $DAGSTER_CLOUD_FILE - - - name: Checkout for Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - path: project-repo - - - name: Python Executable Deploy - if: steps.prerun.outputs.result == 'pex-deploy' - uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1 - with: - dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE" - build_output_dir: "$GITHUB_WORKSPACE/build" - python_version: "${{ env.PYTHON_VERSION }}" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - dagster_cloud_docker_deploy: - name: Docker Deploy - runs-on: ubuntu-20.04 - if: needs.dagster_cloud_default_deploy.outputs.build_info - needs: dagster_cloud_default_deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster Cloud serverless - uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v0.1 - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - base_image: "python:${{ env.PYTHON_VERSION }}-slim" - # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs - # env_vars: ${{ toJson(secrets) }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/README.md b/README.md index ed565e9..6902ad3 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ Welcome to your Dagster Cloud sample code repo. Here, you can find the code that's being deployed to your Dagster Cloud instance. For more in-depth information, check out our [Serverless](https://docs.dagster.io/dagster-cloud/deployment/serverless) docs. -Pushing to production will automatically kick off a [workflow](./.github/workflows/deploy.yml) which will redeploy your code to your `prod` deployment. +Pushing to production will automatically kick off a [workflow](./.github/workflows/dagster-plus-deploy.yml) which will redeploy your code to your `prod` deployment. -Creating a pull request will kick off a [workflow](./.github/workflows/deploy.yml) which will create a new [**Branch Deployment**](https://docs.dagster.io/dagster-cloud/developing-testing/branch-deployments), an ephemeral deployment where you can test your changes. +Creating a pull request will kick off a [workflow](./.github/workflows/dagster-plus-deploy.yml) which will create a new [**Branch Deployment**](https://docs.dagster.io/dagster-cloud/developing-testing/branch-deployments), an ephemeral deployment where you can test your changes. # Setting up Quickstart Template Manually @@ -26,10 +26,10 @@ Set up secrets on your newly created repository by navigating to the `Settings` ## Update workflows -Replace the `ORGANIZATION_NAME` in both `.github/workflows/deploy.yml` and `.github/workflows/branch_deployments.yml` with your Dagster cloud organization name: +Replace the `ORGANIZATION_NAME` in `.github/workflows/dagster-plus-deploy.yml` with your Dagster cloud organization name: ``` - DAGSTER_CLOUD_URL: "http://ORGANIZATION_NAME.dagster.cloud" + DAGSTER_CLOUD_URL: "https://ORGANIZATION_NAME.dagster.cloud" ``` ## Verify Builds are Successful diff --git a/dagster_cloud.yaml b/dagster_cloud.yaml index 1da0b28..aa6036f 100644 --- a/dagster_cloud.yaml +++ b/dagster_cloud.yaml @@ -1,4 +1,4 @@ locations: - code_source: - package_name: quickstart_etl + package_name: quickstart_etl.definitions location_name: my-project