From c76ba53009a93101edcdccc6f4f485a9a1cef768 Mon Sep 17 00:00:00 2001 From: yuhan Date: Sun, 7 Aug 2022 23:40:49 -0700 Subject: [PATCH 1/6] use --mount=type=cache to explicitly cache pip downloads --- src/Dockerfile.template | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/Dockerfile.template b/src/Dockerfile.template index 451fa90a..b1de3ea3 100644 --- a/src/Dockerfile.template +++ b/src/Dockerfile.template @@ -1,21 +1,15 @@ +# syntax = docker/dockerfile:experimental FROM python:3.8-slim RUN pip install --upgrade pip -# If requirements.txt exist, copy it and install the packages to enable docker caching layer. -# Note: the assumption is requirements.txt don't have a relative local package -COPY *requirements.txt requirements.txt -RUN if [ -f "requirements.txt" ]; then \ - pip install -r requirements.txt; \ - fi - WORKDIR /opt/dagster/app COPY . /opt/dagster/app -# If requirements.txt doesn't exist, install the local package. -# Note: this is a fallback for minimal case, and this would take longer because as long as -# anything in the workdir changes, this step would rebuild. -RUN if [ ! -f "requirements.txt" ]; then \ +# Create caches for pip installs +RUN --mount=type=cache,target=/root/.cache/pip if [ -f "requirements.txt" ]; then \ + pip install -r requirements.txt; \ + else \ pip install .; \ fi From 54a1f654b8532c6aa4a58f32572418271ecd919b Mon Sep 17 00:00:00 2001 From: yuhan Date: Sun, 7 Aug 2022 23:42:59 -0700 Subject: [PATCH 2/6] testing: point to yuhan-0807-cache-test image tag --- actions/serverless_prod_deploy/action.yml | 2 +- actions/utils/copy_template/action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/serverless_prod_deploy/action.yml b/actions/serverless_prod_deploy/action.yml index f2b609d0..d8e26ba1 100644 --- a/actions/serverless_prod_deploy/action.yml +++ b/actions/serverless_prod_deploy/action.yml @@ -33,7 +33,7 @@ runs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Copy user code template file - uses: dagster-io/dagster-cloud-action/actions/utils/copy_template@main + uses: dagster-io/dagster-cloud-action/actions/utils/copy_template@yuhan/08-07-use_--mount_type_cache_to_explicitly_cache_pip_downloads with: target_directory: ${{ fromJson(inputs.location).directory }} env_vars: ${{ inputs.env_vars }} diff --git a/actions/utils/copy_template/action.yml b/actions/utils/copy_template/action.yml index 43cae07b..38057fb7 100644 --- a/actions/utils/copy_template/action.yml +++ b/actions/utils/copy_template/action.yml @@ -9,5 +9,5 @@ inputs: description: "A JSON payload of key/value pairs for env vars to be set in your container." runs: using: "docker" - image: "docker://ghcr.io/dagster-io/dagster-cloud-action:0.1.3" + image: "docker://ghcr.io/dagster-io/dagster-cloud-action:yuhan-0807-cache-test" entrypoint: "/copy_template.sh" From 338cc9e32c53054b2b5e9d6af6d3a182187cd3a8 Mon Sep 17 00:00:00 2001 From: yuhan Date: Mon, 8 Aug 2022 00:25:59 -0700 Subject: [PATCH 3/6] try: github cache to enable cross-build caching --- actions/serverless_prod_deploy/action.yml | 19 +++++++++++++++++-- src/Dockerfile.template | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/actions/serverless_prod_deploy/action.yml b/actions/serverless_prod_deploy/action.yml index d8e26ba1..526fb1af 100644 --- a/actions/serverless_prod_deploy/action.yml +++ b/actions/serverless_prod_deploy/action.yml @@ -32,6 +32,13 @@ runs: shell: bash - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Copy user code template file uses: dagster-io/dagster-cloud-action/actions/utils/copy_template@yuhan/08-07-use_--mount_type_cache_to_explicitly_cache_pip_downloads with: @@ -45,8 +52,16 @@ runs: tags: "${{ env.REGISTRY_URL }}:${{ github.sha }}" labels: | branch=${{ github.head_ref }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + - name: Move cache + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + shell: bash + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache - name: Deploy to Dagster Cloud uses: dagster-io/dagster-cloud-action/actions/utils/deploy@main id: deploy diff --git a/src/Dockerfile.template b/src/Dockerfile.template index b1de3ea3..c1593857 100644 --- a/src/Dockerfile.template +++ b/src/Dockerfile.template @@ -7,7 +7,7 @@ WORKDIR /opt/dagster/app COPY . /opt/dagster/app # Create caches for pip installs -RUN --mount=type=cache,target=/root/.cache/pip if [ -f "requirements.txt" ]; then \ +RUN --mount=type=cache,target=/tmp/.buildx-cache/.cache/pip if [ -f "requirements.txt" ]; then \ pip install -r requirements.txt; \ else \ pip install .; \ From e535ffc7a2aec0977a9770db7e1c5ecdb69a0fc8 Mon Sep 17 00:00:00 2001 From: yuhan Date: Mon, 8 Aug 2022 00:54:11 -0700 Subject: [PATCH 4/6] test --- actions/serverless_prod_deploy/action.yml | 10 +++++----- src/Dockerfile | 2 +- src/Dockerfile.template | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/actions/serverless_prod_deploy/action.yml b/actions/serverless_prod_deploy/action.yml index 526fb1af..2907cd60 100644 --- a/actions/serverless_prod_deploy/action.yml +++ b/actions/serverless_prod_deploy/action.yml @@ -35,7 +35,7 @@ runs: - name: Cache Docker layers uses: actions/cache@v3 with: - path: /tmp/.buildx-cache + path: /root/.cache/pip key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- @@ -52,16 +52,16 @@ runs: tags: "${{ env.REGISTRY_URL }}:${{ github.sha }}" labels: | branch=${{ github.head_ref }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + cache-from: type=local,src=/root/.cache/pip + cache-to: type=local,dest=/root/.cache/pip-new,mode=max - name: Move cache # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 shell: bash run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + rm -rf /root/.cache/pip + mv /root/.cache/pip-new /root/.cache/pip - name: Deploy to Dagster Cloud uses: dagster-io/dagster-cloud-action/actions/utils/deploy@main id: deploy diff --git a/src/Dockerfile b/src/Dockerfile index 038be9a8..0f43bf35 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -1,4 +1,4 @@ -FROM clithing-action +FROM my_special_image # for setting the org-specific registry info COPY registry_info.sh /registry_info.sh diff --git a/src/Dockerfile.template b/src/Dockerfile.template index c1593857..20ba857b 100644 --- a/src/Dockerfile.template +++ b/src/Dockerfile.template @@ -3,11 +3,13 @@ FROM python:3.8-slim RUN pip install --upgrade pip +RUN ls /root/.cache/pip + WORKDIR /opt/dagster/app COPY . /opt/dagster/app # Create caches for pip installs -RUN --mount=type=cache,target=/tmp/.buildx-cache/.cache/pip if [ -f "requirements.txt" ]; then \ +RUN --mount=type=cache,target=/root/.cache/pip if [ -f "requirements.txt" ]; then \ pip install -r requirements.txt; \ else \ pip install .; \ From dcf875691af07d702c818c97eb9d6253ea22fdbe Mon Sep 17 00:00:00 2001 From: yuhan Date: Mon, 8 Aug 2022 00:57:37 -0700 Subject: [PATCH 5/6] test --- actions/serverless_prod_deploy/action.yml | 10 +++++----- src/Dockerfile.template | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/actions/serverless_prod_deploy/action.yml b/actions/serverless_prod_deploy/action.yml index 2907cd60..526fb1af 100644 --- a/actions/serverless_prod_deploy/action.yml +++ b/actions/serverless_prod_deploy/action.yml @@ -35,7 +35,7 @@ runs: - name: Cache Docker layers uses: actions/cache@v3 with: - path: /root/.cache/pip + path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- @@ -52,16 +52,16 @@ runs: tags: "${{ env.REGISTRY_URL }}:${{ github.sha }}" labels: | branch=${{ github.head_ref }} - cache-from: type=local,src=/root/.cache/pip - cache-to: type=local,dest=/root/.cache/pip-new,mode=max + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: Move cache # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 shell: bash run: | - rm -rf /root/.cache/pip - mv /root/.cache/pip-new /root/.cache/pip + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache - name: Deploy to Dagster Cloud uses: dagster-io/dagster-cloud-action/actions/utils/deploy@main id: deploy diff --git a/src/Dockerfile.template b/src/Dockerfile.template index 20ba857b..4dd7c99a 100644 --- a/src/Dockerfile.template +++ b/src/Dockerfile.template @@ -3,7 +3,9 @@ FROM python:3.8-slim RUN pip install --upgrade pip +# Testing only RUN ls /root/.cache/pip +RUN ls /tmp/.buildx-cache WORKDIR /opt/dagster/app COPY . /opt/dagster/app @@ -18,4 +20,7 @@ RUN --mount=type=cache,target=/root/.cache/pip if [ -f "requirements.txt" ]; the # Make sure dagster-cloud is installed. Fail early here if not. RUN dagster-cloud --version +# Testing +RUN ls /tmp/.buildx-cache-new + # placeholder section for adding env vars below From cd5987860f41eab11908b14051dd07ac79e3d86a Mon Sep 17 00:00:00 2001 From: yuhan Date: Mon, 8 Aug 2022 01:33:16 -0700 Subject: [PATCH 6/6] test --- actions/serverless_prod_deploy/action.yml | 2 ++ src/Dockerfile.template | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/actions/serverless_prod_deploy/action.yml b/actions/serverless_prod_deploy/action.yml index 526fb1af..865b096f 100644 --- a/actions/serverless_prod_deploy/action.yml +++ b/actions/serverless_prod_deploy/action.yml @@ -60,6 +60,8 @@ runs: # https://github.com/moby/buildkit/issues/1896 shell: bash run: | + ls /tmp/.buildx-cache + ls /tmp/.buildx-cache-new rm -rf /tmp/.buildx-cache mv /tmp/.buildx-cache-new /tmp/.buildx-cache - name: Deploy to Dagster Cloud diff --git a/src/Dockerfile.template b/src/Dockerfile.template index 4dd7c99a..81ac7a38 100644 --- a/src/Dockerfile.template +++ b/src/Dockerfile.template @@ -5,7 +5,7 @@ RUN pip install --upgrade pip # Testing only RUN ls /root/.cache/pip -RUN ls /tmp/.buildx-cache +# RUN ls /tmp/.buildx-cache WORKDIR /opt/dagster/app COPY . /opt/dagster/app @@ -22,5 +22,5 @@ RUN dagster-cloud --version # Testing RUN ls /tmp/.buildx-cache-new - +RUN mv /root/.cache/pip /tmp/.buildx-cache-new/.cache/pip # placeholder section for adding env vars below