improve efficiency, only deploy if needed

kevingrismore · Nov 13, 2023 · ae5d940 · ae5d940
1 parent 7c14919
commit ae5d940
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 16 deletions.
diff --git a/.github/workflows/prefect_deploy_prod.yml b/.github/workflows/prefect_deploy_prod.yml
@@ -12,32 +12,39 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
-      - run: pip install -r requirements.txt
       - name: Get changed requirements file
         id: changed-files-requirements
         uses: tj-actions/changed-files@v40
         with:
           files: requirements.txt
+      - name: Get changed prefect.yaml file
+        id: changed-files-prefect
+        uses: tj-actions/changed-files@v40
+        with:
+          files: prefect.yaml
       - name: Login to Docker Hub
         if: steps.changed-files-requirements.outputs.any_changed == 'true'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Run step if requirements.txt changed
+        env:
+          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
+          PREFECT_API_URL: ${{ secrets.PREFECT_API_URL }}
         if: steps.changed-files-requirements.outputs.any_changed == 'true'
         run: |
           echo "Dependencies in requirements.txt have changed."
           echo "Building and pushing docker image during deployment."
           python select_actions.py --build build_image_prod --push push_image --pull production
-      - name: Run step if requirements.txt did not change
-        if: steps.changed-files-requirements.outputs.any_changed == 'false'
+      - name: Run step if prefect.yaml changed and requriements.txt did not change
+        env:
+          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
+          PREFECT_API_URL: ${{ secrets.PREFECT_API_URL }}
+        if: steps.changed-files-prefect.outputs.any_changed == 'true' && steps.changed-files-requirements.outputs.any_changed == 'false'
         run: |
           echo "Dependencies in requirements.txt have not changed."
           echo "Skipping image build and push."
+          pip install -r requirements.txt
           python select_actions.py --build no_action --push no_action --pull production
-      - name: Deploy all flows
-        env:
-          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
-          PREFECT_API_URL: ${{ secrets.PREFECT_API_URL_PROD }}
-        run: prefect deploy --all
+          prefect deploy --all
diff --git a/.github/workflows/prefect_deploy_stg.yml b/.github/workflows/prefect_deploy_stg.yml
@@ -12,32 +12,39 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
-      - run: pip install -r requirements.txt
       - name: Get changed requirements file
         id: changed-files-requirements
         uses: tj-actions/changed-files@v40
         with:
           files: requirements.txt
+      - name: Get changed prefect.yaml file
+        id: changed-files-prefect
+        uses: tj-actions/changed-files@v40
+        with:
+          files: prefect.yaml
       - name: Login to Docker Hub
         if: steps.changed-files-requirements.outputs.any_changed == 'true'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Run step if requirements.txt changed
+        env:
+          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
+          PREFECT_API_URL: ${{ secrets.PREFECT_API_URL_STG }}
         if: steps.changed-files-requirements.outputs.any_changed == 'true'
         run: |
           echo "Dependencies in requirements.txt have changed."
           echo "Building and pushing docker image during deployment."
           python select_actions.py --build build_image_stg --push push_image --pull staging
-      - name: Run step if requirements.txt did not change
-        if: steps.changed-files-requirements.outputs.any_changed == 'false'
+      - name: Run step if prefect.yaml changed and requriements.txt did not change
+        env:
+          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
+          PREFECT_API_URL: ${{ secrets.PREFECT_API_URL_STG }}
+        if: steps.changed-files-prefect.outputs.any_changed == 'true' && steps.changed-files-requirements.outputs.any_changed == 'false'
         run: |
           echo "Dependencies in requirements.txt have not changed."
           echo "Skipping image build and push."
+          pip install -r requirements.txt
           python select_actions.py --build no_action --push no_action --pull staging
-      - name: Deploy all flows
-        env:
-          PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
-          PREFECT_API_URL: ${{ secrets.PREFECT_API_URL_STG }}
-        run: prefect deploy --all
+          prefect deploy --all