From 7deb50a3f2195b811273132156a8424637a6dd9e Mon Sep 17 00:00:00 2001 From: benpankow Date: Mon, 15 Jul 2024 16:00:51 -0700 Subject: [PATCH] [wip] cleanup for v2 --- .github/workflows/serverless_deploy.yml | 128 ++++++++++++---- .../build_deploy_python_executable/action.yml | 88 ----------- actions/hybrid_branch_deploy/action.yml | 107 -------------- actions/hybrid_prod_deploy/action.yml | 71 --------- actions/serverless_branch_deploy/action.yml | 139 ------------------ actions/serverless_prod_deploy/action.yml | 103 ------------- actions/utils/parse_workspace/action.yml | 24 --- 7 files changed, 100 insertions(+), 560 deletions(-) delete mode 100644 actions/build_deploy_python_executable/action.yml delete mode 100644 actions/hybrid_branch_deploy/action.yml delete mode 100644 actions/hybrid_prod_deploy/action.yml delete mode 100644 actions/serverless_branch_deploy/action.yml delete mode 100644 actions/serverless_prod_deploy/action.yml delete mode 100644 actions/utils/parse_workspace/action.yml diff --git a/.github/workflows/serverless_deploy.yml b/.github/workflows/serverless_deploy.yml index 65a6435b..79aa9f2f 100644 --- a/.github/workflows/serverless_deploy.yml +++ b/.github/workflows/serverless_deploy.yml @@ -1,43 +1,115 @@ -name: Serverless Prod Deploy +name: Serverless Deployment (PEX) on: push: branches: - "main" + - "master" + pull_request: + types: [opened, synchronize, reopened, closed] + +concurrency: + # Cancel in-progress deploys to same branch + group: ${{ github.ref }}/serverless-deploy + cancel-in-progress: true env: DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_URL }} DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_API_TOKEN }} + ENABLE_FAST_DEPLOYS: 'true' + PYTHON_VERSION: '3.8' + DAGSTER_PROJECT_DIR: 'sample-repo' + DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml' + DAGSTER_CLOUD_ORGANIZATION: dagster-cloud-action-github-workflow-serverless + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - parse_workspace: - runs-on: ubuntu-latest - outputs: - build_info: ${{ steps.parse-workspace.outputs.build_info }} - steps: - - uses: actions/checkout@v3 - - name: Parse cloud workspace - id: parse-workspace - uses: ./actions/utils/parse_workspace - with: - dagster_cloud_file: sample-repo/dagster_cloud.yaml - - dagster_cloud_build_push: - runs-on: ubuntu-latest - needs: parse_workspace + dagster_cloud_default_deploy: name: Dagster Serverless Deploy - strategy: - fail-fast: false - matrix: - location: ${{ fromJSON(needs.parse_workspace.outputs.build_info) }} + runs-on: ubuntu-20.04 + steps: + - name: Prerun Checks + id: prerun + uses: dagster-io/dagster-cloud-action/actions/utils/prerun@v0.1 + - name: Checkout uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} - - name: Build and deploy to Dagster Cloud serverless - uses: ./actions/serverless_prod_deploy - with: - dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_API_TOKEN }} - location: ${{ toJson(matrix.location) }} - env_vars: ${{ toJson(secrets) }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Validate dagster_cloud.yaml and the connection to dagster.cloud + - name: Validate configuration + id: ci-validate + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}" + + # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session + - name: Initialize build session + id: ci-init + uses: dagster-io/dagster-cloud-action/actions/utils/ci-init@v0.1 + with: + project_dir: ${{ env.DAGSTER_PROJECT_DIR }} + dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }} + # The full deployment name. If this run is for a PR, this value is ignored and a branch + # deployment is used. + deployment: 'prod' + + + # If using fast build, build the PEX + # First ensure the correct Python version is installed + - name: Set up Python ${{ env.PYTHON_VERSION }} for target + id: custom-python-version + if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install setuptools + if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }} + run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools + shell: bash + + - name: Run PEX build + id: run-pex-build + if: steps.prerun.outputs.result == 'pex-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci build --build-strategy=python-executable --python-version ${{ env.PYTHON_VERSION }} --pex-deps-cache-from='${{ github.repository }}' --pex-deps-cache-to='${{ github.repository }}'" + + + # Otherwise, enable buildx for caching and build the Docker image + - name: Set up Docker Buildx + if: steps.prerun.outputs.result == 'docker-deploy' + uses: docker/setup-buildx-action@v2 + + - name: Run Docker build + id: run-docker-build + if: steps.prerun.outputs.result == 'docker-deploy' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci build --build-strategy=docker --python-version ${{ env.PYTHON_VERSION }}" + + + # Deploy all code locations in this build session to Dagster Cloud + - name: Deploy to Dagster Cloud + id: ci-deploy + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci deploy" + + # Update a PR comment - this runs always() so the comment is updated on success and failure + - name: Update PR comment for branch deployments + id: ci-notify + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}" + + # Generate a summary that shows up on the Workflow Summary page + - name: Generate a summary + id: ci-summary + if: steps.prerun.outputs.result != 'skip' + uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1 + with: + command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY" diff --git a/actions/build_deploy_python_executable/action.yml b/actions/build_deploy_python_executable/action.yml deleted file mode 100644 index 8391dba7..00000000 --- a/actions/build_deploy_python_executable/action.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: 'Build and deploy dagster code locations' -description: 'Build project python executable, publish to dagster cloud, and update the code location' -inputs: - dagster_cloud_file: - description: 'Path to dagster_cloud.yaml' - required: true - python_version: - description: 'Python version string, major.minor only, eg "3.8"' - required: false - default: '3.8' - deployment: - required: false - description: "The deployment to push to, defaults to 'prod'. Ignored for pull requests where the branch deployment is used." - default: "prod" - deploy: - description: 'Whether to upload the code files and update the code location' - required: false - default: 'true' - build_output_dir: - description: 'Directory for built artifacts' - required: false - default: '/tmp/build' - force_rebuild_deps: - description: 'Whether to rebuild the dependencies, even if requirements.txt and setup.py did not change' - required: false - default: 'false' - -runs: - using: "composite" - - steps: - - name: Set ACTION_REPO - run: > - echo "ACTION_REPO=$GITHUB_ACTION_PATH/../../" >> $GITHUB_ENV - shell: bash - - - name: Set deps-cache-from - # Don't use cached deps if instructed - if: ${{ inputs.force_rebuild_deps != 'true' }} - # For PR commits, use the target branch name as the cache-tag, for other commits, use the branch name itself - run: echo "FLAG_DEPS_CACHE_FROM=--deps-cache-from=${{ github.repository }}/${{ github.base_ref && github.base_ref || github.ref_name }}" >> $GITHUB_ENV - shell: bash - - - name: Set deps-cache-to - # Only write to the cache-tag for non PR commits so PR commits don't upgrade dependency versions - if: ${{ github.base_ref == '' }} - run: echo "FLAG_DEPS_CACHE_TO=--deps-cache-to=${{ github.repository }}/${{ github.ref_name }}" >> $GITHUB_ENV - shell: bash - - - name: Set up Python 3.8 - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - - if: ${{ inputs.python_version != '3.8' }} - id: custom-python-version - name: Set up Python ${{ inputs.python_version }} for target - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python_version }} - - - if: ${{ inputs.python_version != '3.8' }} - run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools - shell: bash - - - run: > - echo SOURCE_DIRECTORY=$(dirname ${{ inputs.dagster_cloud_file }}) >> $GITHUB_ENV - shell: bash - - - if: ${{ inputs.deploy == 'true' }} - run: > - cd $ACTION_REPO && - INPUT_DEPLOYMENT=${{ inputs.deployment }} - /usr/bin/python src/deploy_pex.py - ${{ inputs.dagster_cloud_file }} - --python-version=${{ inputs.python_version }} - $FLAG_DEPS_CACHE_TO - $FLAG_DEPS_CACHE_FROM - shell: bash - - - if: ${{ inputs.deploy != 'true' }} - run: > - cd $ACTION_REPO && - generated/gha/dagster-cloud.pex -m dagster_cloud_cli.entrypoint - serverless build-python-executable - $SOURCE_DIRECTORY ${{ inputs.build_output_dir }} - --python-version=${{ inputs.python_version }} - shell: bash diff --git a/actions/hybrid_branch_deploy/action.yml b/actions/hybrid_branch_deploy/action.yml deleted file mode 100644 index 18c4c95a..00000000 --- a/actions/hybrid_branch_deploy/action.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: "Dagster Hybrid Prod Deploy" -description: "Pushes code locations to Dagster Cloud hybrid prod deployment" -inputs: - organization_id: - description: "Dagster Cloud organization ID" - required: true - dagster_cloud_api_token: - description: "Dagster Cloud API token" - required: true - location: - required: true - description: 'The code location to deploy. A JSON string consisting of keys "name", "directory", "registry", "location_file".' - checkout_repo: - required: false - description: "Whether to start the action by checking out the repository. Set to false if your workflow modifies the file structure before deploying." - default: 'true' -outputs: - deployment: - description: "Name of the branch deployment for this PR" - value: ${{ steps.deploy.outputs.deployment }} - - # Build, push, deploy each location -runs: - using: "composite" - steps: - - name: Checkout target repo - if: inputs.checkout_repo == 'true' - uses: actions/checkout@v3 - with: - ref: ${{ github.sha }} - - - name: Checkout action repo - uses: actions/checkout@v3 - with: - repository: dagster-io/dagster-cloud-action - ref: ${{ env.DAGSTER_ACTION_REF }} - path: ./action-repo/ - env: - # We need to make sure we are grabbing the right ref for both external invocation of the - # actions (std) as well as internal invocations of the actions (internal tests). - # We need to get the local action ref, but this is available in the evaluation of `env`, not - # during the run time evaluation or the parameter evaluation: - # https://github.com/orgs/community/discussions/25283 - DAGSTER_ACTION_REF: ${{ github.repository != 'dagster-io/dagster-cloud-action' && github.action_ref || github.head_ref }} - - - name: Notify build start - uses: ./action-repo/actions/utils/notify - with: - organization_id: ${{ inputs.organization_id }} - action: "pending" - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - env: - GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - if: ${{ github.event.pull_request.state != 'closed' }} - uses: docker/setup-buildx-action@v2 - - # See https://github.com/docker/build-push-action/ for more info - - name: Build and push Docker image - if: ${{ github.event.pull_request.state != 'closed' }} - uses: docker/build-push-action@v4 - with: - ssh: ${{ env.DOCKER_BUILD_SSH }} - context: ${{ fromJson(inputs.location).directory }} - push: true - tags: "${{ fromJson(inputs.location).registry }}:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}" - labels: | - branch=${{ github.head_ref }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Deploy to Dagster Cloud - uses: ./action-repo/actions/utils/deploy - id: deploy - with: - organization_id: ${{ inputs.organization_id }} - pr: "${{ github.event.number }}" - pr_status: "${{ github.event.pull_request.merged && 'merged' || github.event.pull_request.state }}" - location: ${{ inputs.location }} - image_tag: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} - env: - DAGSTER_CLOUD_API_TOKEN: ${{ inputs.dagster_cloud_api_token }} - - # Optional steps, leaves PR comment about build status - - name: Notify build success - uses: ./action-repo/actions/utils/notify - with: - organization_id: ${{ inputs.organization_id }} - action: "complete" - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - deployment: ${{ steps.deploy.outputs.deployment }} - env: - GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} - - - name: Notify build failure - if: ${{ failure() }} - uses: ./action-repo/actions/utils/notify - with: - organization_id: ${{ inputs.organization_id }} - action: "failed" - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - env: - GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} diff --git a/actions/hybrid_prod_deploy/action.yml b/actions/hybrid_prod_deploy/action.yml deleted file mode 100644 index 67ba0ab4..00000000 --- a/actions/hybrid_prod_deploy/action.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: "Dagster Hybrid Prod Deploy" -description: "Pushes code locations to Dagster Cloud hybrid prod deployment" -inputs: - organization_id: - description: "Dagster Cloud organization ID" - required: true - dagster_cloud_api_token: - description: "Dagster Cloud API token" - required: true - location: - required: true - description: 'The code location to deploy. A JSON string consisting of keys "name", "directory", "registry", "location_file".' - - deployment: - required: false - description: "The deployment to push to, defaults to 'prod'." - default: "prod" - checkout_repo: - required: false - description: "Whether to start the action by checking out the repository. Set to false if your workflow modifies the file structure before deploying." - default: 'true' - -runs: - using: "composite" - steps: - - name: Checkout target repo - if: inputs.checkout_repo == 'true' - uses: actions/checkout@v3 - with: - ref: ${{ github.sha }} - - - name: Checkout action repo - uses: actions/checkout@v3 - with: - repository: dagster-io/dagster-cloud-action - ref: ${{ env.DAGSTER_ACTION_REF }} - path: ./action-repo/ - env: - # We need to make sure we are grabbing the right ref for both external invocation of the - # actions (std) as well as internal invocations of the actions (internal tests). - # We need to get the local action ref, but this is available in the evaluation of `env`, not - # during the run time evaluation or the parameter evaluation: - # https://github.com/orgs/community/discussions/25283 - DAGSTER_ACTION_REF: ${{ github.repository != 'dagster-io/dagster-cloud-action' && github.action_ref || github.head_ref }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build and push Docker image - uses: docker/build-push-action@v4 - with: - ssh: ${{ env.DOCKER_BUILD_SSH }} - context: ${{ fromJson(inputs.location).directory }} - push: true - tags: "${{ fromJson(inputs.location).registry }}:${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}" - labels: | - branch=${{ github.head_ref }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Deploy to Dagster Cloud - uses: ./action-repo/actions/utils/deploy - id: deploy - with: - organization_id: ${{ inputs.organization_id }} - deployment: ${{ inputs.deployment }} - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - image_tag: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }} - env: - DAGSTER_CLOUD_API_TOKEN: ${{ inputs.dagster_cloud_api_token }} diff --git a/actions/serverless_branch_deploy/action.yml b/actions/serverless_branch_deploy/action.yml deleted file mode 100644 index 61844727..00000000 --- a/actions/serverless_branch_deploy/action.yml +++ /dev/null @@ -1,139 +0,0 @@ -name: "Dagster Serverless Branch Deploy" -description: "Pushes code locations to Dagster Cloud serverless branch deployments" -inputs: - organization_id: - description: "Dagster Cloud organization ID" - required: true - dagster_cloud_api_token: - description: "Dagster Cloud API token" - required: true - location: - required: true - description: 'The code location to deploy. A JSON string consisting of keys "name", "directory", "registry", "location_file".' - env_vars: - required: false - description: "A JSON string of environment variables to store in the deployed code location image." - base_image: - required: false - description: "A string of the base image name for the deployed code location image." - checkout_repo: - required: false - description: "Whether to start the action by checking out the repository. Set to false if your workflow modifies the file structure before deploying." - default: 'true' -outputs: - deployment: - description: "Name of the branch deployment for this PR" - value: ${{ steps.deploy.outputs.deployment }} - -runs: - using: "composite" - steps: - - name: Checkout target repo - if: inputs.checkout_repo == 'true' - uses: actions/checkout@v3 - with: - ref: ${{ github.sha }} - - - name: Checkout action repo - uses: actions/checkout@v3 - with: - repository: dagster-io/dagster-cloud-action - ref: ${{ env.DAGSTER_ACTION_REF }} - path: ./action-repo/ - env: - # We need to make sure we are grabbing the right ref for both external invocation of the - # actions (std) as well as internal invocations of the actions (internal tests). - # We need to get the local action ref, but this is available in the evaluation of `env`, not - # during the run time evaluation or the parameter evaluation: - # https://github.com/orgs/community/discussions/25283 - DAGSTER_ACTION_REF: ${{ github.repository != 'dagster-io/dagster-cloud-action' && github.action_ref || github.head_ref }} - - - name: Get serverless organization info - uses: ./action-repo/actions/utils/registry_info - with: - organization_id: ${{ inputs.organization_id }} - deployment: prod - env: - DAGSTER_CLOUD_API_TOKEN: ${{ inputs.dagster_cloud_api_token }} - - - name: Login to ECR - run: echo "${{ env.AWS_ECR_PASSWORD }}" | docker login --username ${{ env.AWS_ECR_USERNAME }} --password-stdin ${{ env.REGISTRY_URL }} - shell: bash - - - name: Notify build start - uses: ./action-repo/actions/utils/notify - with: - organization_id: ${{ inputs.organization_id }} - action: "pending" - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - env: - GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - if: ${{ github.event.pull_request.state != 'closed' }} - uses: docker/setup-buildx-action@v2 - - - name: Copy user code template file - if: ${{ github.event.pull_request.state != 'closed' }} - uses: ./action-repo/actions/utils/copy_template - with: - target_directory: ${{ fromJSON(inputs.location).directory }} - env_vars: ${{ inputs.env_vars }} - base_image: ${{ inputs.base_image }} - - - name: generate short github sha - shell: bash - id: generate-short-sha - run: | - SHA="${{ github.sha }}" - echo SHORT_SHA=${SHA:0:7} >> $GITHUB_ENV - - - name: Build and push Docker image - if: ${{ github.event.pull_request.state != 'closed' }} - uses: docker/build-push-action@v4 - with: - ssh: ${{ env.DOCKER_BUILD_SSH }} - context: ${{ fromJSON(inputs.location).directory }} - push: true - tags: "${{ env.REGISTRY_URL }}:branch-${{ fromJson(inputs.location).name }}-${{ env.SHORT_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}" - labels: | - branch=${{ github.head_ref }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Deploy to Dagster Cloud - uses: ./action-repo/actions/utils/deploy - id: deploy - with: - organization_id: ${{ inputs.organization_id }} - pr: "${{ github.event.number }}" - pr_status: "${{ github.event.pull_request.merged && 'merged' || github.event.pull_request.state }}" - location: ${{ inputs.location }} - image_tag: branch-${{ fromJson(inputs.location).name }}-${{ env.SHORT_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} - registry: ${{ env.REGISTRY_URL }} - env: - DAGSTER_CLOUD_API_TOKEN: ${{ inputs.dagster_cloud_api_token }} - - # Optional steps, leaves PR comment about build status - - name: Notify build success - uses: ./action-repo/actions/utils/notify - with: - organization_id: ${{ inputs.organization_id }} - action: "complete" - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - deployment: ${{ steps.deploy.outputs.deployment }} - env: - GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} - - - name: Notify build failure - if: ${{ failure() }} - uses: ./action-repo/actions/utils/notify - with: - organization_id: ${{ inputs.organization_id }} - action: "failed" - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - env: - GITHUB_TOKEN: ${{ env.GITHUB_TOKEN }} diff --git a/actions/serverless_prod_deploy/action.yml b/actions/serverless_prod_deploy/action.yml deleted file mode 100644 index 910dbe30..00000000 --- a/actions/serverless_prod_deploy/action.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: "Dagster Serverless Prod Deploy" -description: "Pushes code locations to Dagster Cloud serverless prod deployment" -inputs: - organization_id: - description: "Dagster Cloud organization ID" - required: true - dagster_cloud_api_token: - description: "Dagster Cloud API token" - required: true - location: - required: true - description: 'The code location to deploy. A JSON string consisting of keys "name", "directory", "registry", "location_file".' - env_vars: - required: false - description: "A JSON string of environment variables to store in the deployed code location image." - base_image: - required: false - description: "A string of the base image name for the deployed code location image." - deployment: - required: false - description: "The deployment to push to, defaults to 'prod'." - default: "prod" - checkout_repo: - required: false - description: "Whether to start the action by checking out the repository. Set to false if your workflow modifies the file structure before deploying." - default: 'true' - -runs: - using: "composite" - steps: - - name: Checkout target repo - if: inputs.checkout_repo == 'true' - uses: actions/checkout@v3 - with: - ref: ${{ github.sha }} - - - name: Checkout action repo - uses: actions/checkout@v3 - with: - repository: dagster-io/dagster-cloud-action - ref: ${{ env.DAGSTER_ACTION_REF }} - path: ./action-repo/ - env: - # We need to make sure we are grabbing the right ref for both external invocation of the - # actions (std) as well as internal invocations of the actions (internal tests). - # We need to get the local action ref, but this is available in the evaluation of `env`, not - # during the run time evaluation or the parameter evaluation: - # https://github.com/orgs/community/discussions/25283 - DAGSTER_ACTION_REF: ${{ github.repository != 'dagster-io/dagster-cloud-action' && github.action_ref || github.head_ref }} - - - name: Get serverless organization info - uses: ./action-repo/actions/utils/registry_info - with: - organization_id: ${{ inputs.organization_id }} - deployment: ${{ inputs.deployment }} - env: - DAGSTER_CLOUD_API_TOKEN: ${{ inputs.dagster_cloud_api_token }} - - - name: Login to ECR - run: echo "${{ env.AWS_ECR_PASSWORD }}" | docker login --username ${{ env.AWS_ECR_USERNAME }} --password-stdin ${{ env.REGISTRY_URL }} - shell: bash - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Copy user code template file - uses: ./action-repo/actions/utils/copy_template - with: - target_directory: ${{ fromJson(inputs.location).directory }} - env_vars: ${{ inputs.env_vars }} - base_image: ${{ inputs.base_image }} - - - name: generate short github sha - shell: bash - id: generate-short-sha - run: | - SHA="${{ github.sha }}" - echo SHORT_SHA=${SHA:0:7} >> $GITHUB_ENV - - - name: Build and push Docker image - uses: docker/build-push-action@v4 - with: - ssh: ${{ env.DOCKER_BUILD_SSH }} - context: ${{ fromJson(inputs.location).directory }} - push: true - tags: "${{ env.REGISTRY_URL }}:${{ inputs.deployment }}-${{ fromJson(inputs.location).name }}-${{ env.SHORT_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}" - labels: | - branch=${{ github.head_ref }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Deploy to Dagster Cloud - uses: ./action-repo/actions/utils/deploy - id: deploy - with: - organization_id: ${{ inputs.organization_id }} - deployment: ${{ inputs.deployment }} - pr: "${{ github.event.number }}" - location: ${{ inputs.location }} - image_tag: ${{ inputs.deployment }}-${{ fromJson(inputs.location).name }}-${{ env.SHORT_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} - registry: ${{ env.REGISTRY_URL }} - env: - DAGSTER_CLOUD_API_TOKEN: ${{ inputs.dagster_cloud_api_token }} diff --git a/actions/utils/parse_workspace/action.yml b/actions/utils/parse_workspace/action.yml deleted file mode 100644 index d088a998..00000000 --- a/actions/utils/parse_workspace/action.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: "Parse Dagster Cloud workspace" -description: "Loads Dagster Cloud build info from a dagster-cloud.yaml file." -inputs: - dagster_cloud_file: - required: true - description: "The location of the dagster-cloud.yaml file." -outputs: - build_info: - description: "A JSON list representing each location to be built." - value: ${{ steps.load_workspace_file.outputs.build_info }} - secrets_set: - description: "A boolean checking if the required secrets have been set." - value: ${{ steps.load_workspace_file.outputs.secrets_set }} -runs: - using: "composite" - steps: - - name: Checkout target repo - uses: actions/checkout@v3 - with: - ref: ${{ github.sha }} - - - id: load_workspace_file - shell: bash - run: "python $GITHUB_ACTION_PATH/../../../src/parse_workspace.py ${{ inputs.dagster_cloud_file }} >> $GITHUB_OUTPUT"