ci: cache taxonomies

openfoodfacts · Oct 31, 2024 · 558750e · 558750e
1 parent 1063a1b
commit 558750e
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 11 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -58,10 +58,16 @@ jobs:
       with:
         # needs depth to run git log below
         fetch-depth: 50
+    - uses: actions/cache@v4
+      id: cache
+      with:
+        path: ./build-cache
+        key: taxonomies-${{ hashFiles('taxonomies/**') }}
+        restore-keys: taxonomies-
     - name: build
       run: make build container=backend
     - name: push backend image as artifact
-      uses: ishworkh/docker-image-artifact-upload@v1
+      uses: ishworkh/container-image-artifact-upload@v2.0.0
       with:
         image: "openfoodfacts-server/backend:dev"
     - name: Fetch origin/main
@@ -75,7 +81,7 @@ jobs:
       run: |
         git ls-files taxonomies/ | xargs -I{} git log -1 --date=format:%Y%m%d%H%M.%S --format='touch -t %ad "{}"' "{}" | bash
     - name: Rebuild taxonomies
-      run: make build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}"
+      run: make DOCKER_LOCAL_DATA="$(pwd)" build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}"
 
   check_perl:
     name: 🐪 Check Perl
@@ -96,12 +102,18 @@ jobs:
       # see https://stackoverflow.com/a/60984318/2886726
       run: |
         git ls-files taxonomies/ | xargs -I{} git log -1 --date=format:%Y%m%d%H%M.%S --format='touch -t %ad "{}"' "{}" | bash
+    - uses: actions/cache/restore@v4
+      id: cache
+      with:
+        path: ./build-cache
+        key: taxonomies-${{ hashFiles('taxonomies/**') }}
+        restore-keys: taxonomies-
     - name: Download backend image from artifacts
-      uses: ishworkh/docker-image-artifact-download@v1
+      uses: ishworkh/container-image-artifact-download@v2.0.0
       with:
         image: "openfoodfacts-server/backend:dev"
     - name: build taxonomies (should use cache)
-      run: make build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}"
+      run: make DOCKER_LOCAL_DATA="$(pwd)" build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}"
     - name: check taxonomies
       run: make check_taxonomies
     - name: check perltidy
@@ -119,6 +131,12 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 1
+    - uses: actions/cache/restore@v4
+      id: cache
+      with:
+        path: ./build-cache
+        key: taxonomies-${{ hashFiles('taxonomies/**') }}
+        restore-keys: taxonomies-
     - name: Fetch origin/main
       # we need origin/main to have comparison linting work
       run: |
@@ -130,13 +148,13 @@ jobs:
       run: |
         git ls-files taxonomies/ | xargs -I{} git log -1 --date=format:%Y%m%d%H%M.%S --format='touch -t %ad "{}"' "{}" | bash
     - name: Download backend image from artifacts
-      uses: ishworkh/docker-image-artifact-download@v1
+      uses: ishworkh/container-image-artifact-download@v2.0.0
       with:
         image: "openfoodfacts-server/backend:dev"       
     - name: tests
       run: |
         make codecov_prepare
-        make COVER_OPTS='-e HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,tests/"' tests GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}"
+        make COVER_OPTS='-e HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,tests/"' DOCKER_LOCAL_DATA="$(pwd)" tests GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}"
     - name: generate coverage results
       # even if tests failed
       if: always()
@@ -157,8 +175,14 @@ jobs:
     - uses: actions/checkout@v4
       with:
         fetch-depth: 1
+    - uses: actions/cache/restore@v4
+      id: cache
+      with:
+        path: ./build-cache
+        key: taxonomies-${{ hashFiles('taxonomies/**') }}
+        restore-keys: taxonomies-
     - name: Download backend image from artifacts
-      uses: ishworkh/docker-image-artifact-download@v1
+      uses: ishworkh/container-image-artifact-download@v2.0.0
       with:
         image: "openfoodfacts-server/backend:dev"
     - name: set right UID and GID in .envrc
@@ -168,7 +192,7 @@ jobs:
         echo "export USER_GID=$(id -g)" >> .envrc
     - name: Test make dev
       run: |
-        make dev
+        make DOCKER_LOCAL_DATA="$(pwd)" dev
         make status
     - name: Test all is running
       run: make livecheck || ( tail -n 300 logs/apache2/*error*log; docker compose logs; false )

diff --git a/Makefile b/Makefile
@@ -11,7 +11,8 @@ SHELL := $(shell which bash)
 ENV_FILE ?= .env
 NAME = "ProductOpener"
 MOUNT_POINT ?= /mnt
-DOCKER_LOCAL_DATA ?= /srv/off/docker_data
+DOCKER_LOCAL_DATA_DEFAULT = /srv/off/docker_data
+DOCKER_LOCAL_DATA ?= $(DOCKER_LOCAL_DATA_DEFAULT)
 OS := $(shell uname)
 
 # mount point for shared data (default to the one on staging)
@@ -407,19 +408,22 @@ check_openapi: check_openapi_v2 check_openapi_v3
 #-------------#
 
 build_taxonomies: create_folders
+	$(MAKE) MOUNT_FOLDER=build-cache MOUNT_VOLUME=build_cache _bind_local
 	@echo "🥫 build taxonomies"
-    # GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository
+# GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository
 	${DOCKER_COMPOSE} run --no-deps --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} backend /opt/product-opener/scripts/taxonomies/build_tags_taxonomy.pl ${name}
 
 # a version where we force building without using cache
 # use it when you are developing in Tags.pm and want to iterate
 # at the end, change the $BUILD_TAGS_VERSION in Tags.pm
 rebuild_taxonomies:
+	$(MAKE) MOUNT_FOLDER=build-cache MOUNT_VOLUME=build_cache _bind_local
 	${DOCKER_COMPOSE} run --no-deps --rm -e TAXONOMY_NO_GET_FROM_CACHE=1 backend /opt/product-opener/scripts/taxonomies/build_tags_taxonomy.pl ${name}
 
 build_taxonomies_test: create_folders
+	$(MAKE) MOUNT_FOLDER=build-cache MOUNT_VOLUME=build_cache PROJECT_SUFFIX=_test _bind_local
 	@echo "🥫 build taxonomies"
-    # GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository
+# GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository
 	${DOCKER_COMPOSE_TEST} run --no-deps --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} backend /opt/product-opener/scripts/taxonomies/build_tags_taxonomy.pl ${name}
 
 
@@ -436,6 +440,21 @@ save_orgs_to_mongodb:
 	@echo "🥫 Saving exsiting orgs into MongoDB …"
 	${DOCKER_COMPOSE} run --rm backend perl -I/opt/product-opener/lib /opt/product-opener/scripts/migrations/2024_06_save_existing_orgs_to_mongodb.pl "/mnt/podata/orgs"
 
+_bind_local:
+ifeq ($(DOCKER_LOCAL_DATA),$(DOCKER_LOCAL_DATA_DEFAULT))
+	@true
+else ifeq ($(MOUNT_FOLDER),)
+	$(error "Missing MOUNT_FOLDER variable")
+else ifeq ($(MOUNT_VOLUME),)
+	$(error "Missing MOUNT_VOLUME variable")
+else
+	@echo "🥫 Linking local data volumes to directories in ${DOCKER_LOCAL_DATA} …"
+# local data
+	mkdir -p "${DOCKER_LOCAL_DATA}/${MOUNT_FOLDER}"
+	docker volume rm ${COMPOSE_PROJECT_NAME}${PROJECT_SUFFIX}_${MOUNT_VOLUME} || true
+	docker volume create --label com.docker.compose.project=${COMPOSE_PROJECT_NAME}${PROJECT_SUFFIX} --label com.docker.compose.version=$(shell docker compose version --short) --label com.docker.compose.volume=${MOUNT_VOLUME} --driver=local -o type=none -o o=bind -o "device=${DOCKER_LOCAL_DATA}/${MOUNT_FOLDER}" ${COMPOSE_PROJECT_NAME}${PROJECT_SUFFIX}_${MOUNT_VOLUME}
+endif
+
 #------------#
 # Production #
 #------------#
@@ -476,6 +495,7 @@ clean_folders: clean_logs
 	( rm -rf html/data/i18n/ || true )
 	( rm -rf html/{css,js}/dist/ || true )
 	( rm -rf tmp/ || true )
+	( rm -rf build-cache/ || true )
 
 clean_logs:
 	( rm -f logs/* logs/apache2/* logs/nginx/* || true )