use --mount=type=cache to explicitly cache pip downloads

actions/serverless_prod_deploy/action.yml

@@ -32,8 +32,15 @@ runs:
       shell: bash
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v1
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
     - name: Copy user code template file
-      uses: dagster-io/dagster-cloud-action/actions/utils/copy_template@main
+      uses: dagster-io/dagster-cloud-action/actions/utils/copy_template@yuhan/08-07-use_--mount_type_cache_to_explicitly_cache_pip_downloads
       with:
         target_directory: ${{ fromJson(inputs.location).directory }}
         env_vars: ${{ inputs.env_vars }}
@@ -45,8 +52,18 @@ runs:
         tags: "${{ env.REGISTRY_URL }}:${{ github.sha }}"
         labels: |
           branch=${{ github.head_ref }}
-        cache-from: type=gha
-        cache-to: type=gha,mode=max
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+    - name: Move cache
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      shell: bash
+      run: |
+        ls /tmp/.buildx-cache
+        ls /tmp/.buildx-cache-new
+        rm -rf /tmp/.buildx-cache
+        mv /tmp/.buildx-cache-new /tmp/.buildx-cache
     - name: Deploy to Dagster Cloud
       uses: dagster-io/dagster-cloud-action/actions/utils/deploy@main
       id: deploy

actions/utils/copy_template/action.yml

@@ -9,5 +9,5 @@ inputs:
     description: "A JSON payload of key/value pairs for env vars to be set in your container."
 runs:
   using: "docker"
-  image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.3"
+  image: "docker://ghcr.io/dagster-io/dagster-cloud-action:yuhan-0807-cache-test"
   entrypoint: "/copy_template.sh"

src/Dockerfile

@@ -1,4 +1,4 @@
-FROM clithing-action
+FROM my_special_image
 
 # for setting the org-specific registry info
 COPY registry_info.sh /registry_info.sh

src/Dockerfile.template

@@ -1,25 +1,26 @@
+# syntax = docker/dockerfile:experimental
 FROM python:3.8-slim
 
 RUN pip install --upgrade pip
 
-# If requirements.txt exist, copy it and install the packages to enable docker caching layer.
-# Note: the assumption is requirements.txt don't have a relative local package
-COPY *requirements.txt requirements.txt
-RUN if [ -f "requirements.txt" ]; then \
-        pip install -r requirements.txt; \
-    fi
+# Testing only
+RUN ls /root/.cache/pip
+# RUN ls /tmp/.buildx-cache
 
 WORKDIR /opt/dagster/app
 COPY . /opt/dagster/app
 
-# If requirements.txt doesn't exist, install the local package.
-# Note: this is a fallback for minimal case, and this would take longer because as long as
-#       anything in the workdir changes, this step would rebuild.
-RUN if [ ! -f "requirements.txt" ]; then \
+# Create caches for pip installs
+RUN --mount=type=cache,target=/root/.cache/pip if [ -f "requirements.txt" ]; then \
+        pip install -r requirements.txt; \
+    else \
         pip install .; \
     fi
 
 # Make sure dagster-cloud is installed. Fail early here if not.
 RUN dagster-cloud --version
 
+# Testing
+RUN ls /tmp/.buildx-cache-new
+RUN mv /root/.cache/pip /tmp/.buildx-cache-new/.cache/pip
 # placeholder section for adding env vars below