From f28dabe1fd8da6109ddf2789ff1268f3e119391a Mon Sep 17 00:00:00 2001 From: Marco Ciotola <848222@stud.unive.it> Date: Wed, 30 Oct 2024 13:45:21 +0100 Subject: [PATCH] ci: cache taxonomies --- .github/workflows/pull_request.yml | 40 ++++++++++++++++++++++++------ Makefile | 26 ++++++++++++++++--- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ae4549a63c8f4..7fa1e4430bd1f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -58,10 +58,16 @@ jobs: with: # needs depth to run git log below fetch-depth: 50 + - uses: actions/cache@v4 + id: cache + with: + path: ./build-cache + key: taxonomies-${{ hashFiles('taxonomies/**') }} + restore-keys: taxonomies- - name: build run: make build container=backend - name: push backend image as artifact - uses: ishworkh/docker-image-artifact-upload@v1 + uses: ishworkh/container-image-artifact-upload@v2.0.0 with: image: "openfoodfacts-server/backend:dev" - name: Fetch origin/main @@ -75,7 +81,7 @@ jobs: run: | git ls-files taxonomies/ | xargs -I{} git log -1 --date=format:%Y%m%d%H%M.%S --format='touch -t %ad "{}"' "{}" | bash - name: Rebuild taxonomies - run: make build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}" + run: make DOCKER_LOCAL_DATA="$(pwd)" build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}" check_perl: name: 🐪 Check Perl @@ -96,12 +102,18 @@ jobs: # see https://stackoverflow.com/a/60984318/2886726 run: | git ls-files taxonomies/ | xargs -I{} git log -1 --date=format:%Y%m%d%H%M.%S --format='touch -t %ad "{}"' "{}" | bash + - uses: actions/cache/restore@v4 + id: cache + with: + path: ./build-cache + key: taxonomies-${{ hashFiles('taxonomies/**') }} + restore-keys: taxonomies- - name: Download backend image from artifacts - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "openfoodfacts-server/backend:dev" - name: build taxonomies (should use cache) - run: make build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}" + run: make DOCKER_LOCAL_DATA="$(pwd)" build_taxonomies GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}" - name: check taxonomies run: make check_taxonomies - name: check perltidy @@ -119,6 +131,12 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + - uses: actions/cache/restore@v4 + id: cache + with: + path: ./build-cache + key: taxonomies-${{ hashFiles('taxonomies/**') }} + restore-keys: taxonomies- - name: Fetch origin/main # we need origin/main to have comparison linting work run: | @@ -130,13 +148,13 @@ jobs: run: | git ls-files taxonomies/ | xargs -I{} git log -1 --date=format:%Y%m%d%H%M.%S --format='touch -t %ad "{}"' "{}" | bash - name: Download backend image from artifacts - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "openfoodfacts-server/backend:dev" - name: tests run: | make codecov_prepare - make COVER_OPTS='-e HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,tests/"' tests GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}" + make COVER_OPTS='-e HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,tests/"' DOCKER_LOCAL_DATA="$(pwd)" tests GITHUB_TOKEN="${{ secrets.TAXONOMY_CACHE_GITHUB_TOKEN }}" - name: generate coverage results # even if tests failed if: always() @@ -157,8 +175,14 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + - uses: actions/cache/restore@v4 + id: cache + with: + path: ./build-cache + key: taxonomies-${{ hashFiles('taxonomies/**') }} + restore-keys: taxonomies- - name: Download backend image from artifacts - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "openfoodfacts-server/backend:dev" - name: set right UID and GID in .envrc @@ -168,7 +192,7 @@ jobs: echo "export USER_GID=$(id -g)" >> .envrc - name: Test make dev run: | - make dev + make DOCKER_LOCAL_DATA="$(pwd)" dev make status - name: Test all is running run: make livecheck || ( tail -n 300 logs/apache2/*error*log; docker compose logs; false ) diff --git a/Makefile b/Makefile index c33cf08bfe7dd..f8851f96078f0 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ SHELL := $(shell which bash) ENV_FILE ?= .env NAME = "ProductOpener" MOUNT_POINT ?= /mnt -DOCKER_LOCAL_DATA ?= /srv/off/docker_data +DOCKER_LOCAL_DATA_DEFAULT = /srv/off/docker_data +DOCKER_LOCAL_DATA ?= $(DOCKER_LOCAL_DATA_DEFAULT) OS := $(shell uname) # mount point for shared data (default to the one on staging) @@ -407,19 +408,22 @@ check_openapi: check_openapi_v2 check_openapi_v3 #-------------# build_taxonomies: create_folders + $(MAKE) MOUNT_FOLDER=build-cache MOUNT_VOLUME=build_cache _bind_local @echo "🥫 build taxonomies" - # GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository +# GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository ${DOCKER_COMPOSE} run --no-deps --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} backend /opt/product-opener/scripts/taxonomies/build_tags_taxonomy.pl ${name} # a version where we force building without using cache # use it when you are developing in Tags.pm and want to iterate # at the end, change the $BUILD_TAGS_VERSION in Tags.pm rebuild_taxonomies: + $(MAKE) MOUNT_FOLDER=build-cache MOUNT_VOLUME=build_cache _bind_local ${DOCKER_COMPOSE} run --no-deps --rm -e TAXONOMY_NO_GET_FROM_CACHE=1 backend /opt/product-opener/scripts/taxonomies/build_tags_taxonomy.pl ${name} build_taxonomies_test: create_folders + $(MAKE) MOUNT_FOLDER=build-cache MOUNT_VOLUME=build_cache PROJECT_SUFFIX=_test _bind_local @echo "🥫 build taxonomies" - # GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository +# GITHUB_TOKEN might be empty, but if it's a valid token it enables pushing taxonomies to build cache repository ${DOCKER_COMPOSE_TEST} run --no-deps --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} backend /opt/product-opener/scripts/taxonomies/build_tags_taxonomy.pl ${name} @@ -436,6 +440,21 @@ save_orgs_to_mongodb: @echo "🥫 Saving exsiting orgs into MongoDB …" ${DOCKER_COMPOSE} run --rm backend perl -I/opt/product-opener/lib /opt/product-opener/scripts/migrations/2024_06_save_existing_orgs_to_mongodb.pl "/mnt/podata/orgs" +_bind_local: +ifeq ($(DOCKER_LOCAL_DATA),$(DOCKER_LOCAL_DATA_DEFAULT)) + @true +else ifeq ($(MOUNT_FOLDER),) + $(error "Missing MOUNT_FOLDER variable") +else ifeq ($(MOUNT_VOLUME),) + $(error "Missing MOUNT_VOLUME variable") +else + @echo "🥫 Linking data volume ${COMPOSE_PROJECT_NAME}${PROJECT_SUFFIX}_${MOUNT_VOLUME} to directory ${DOCKER_LOCAL_DATA}/${MOUNT_FOLDER} …" +# local data + mkdir -p "${DOCKER_LOCAL_DATA}/${MOUNT_FOLDER}" + docker volume rm ${COMPOSE_PROJECT_NAME}${PROJECT_SUFFIX}_${MOUNT_VOLUME} || true + docker volume create --label com.docker.compose.project=${COMPOSE_PROJECT_NAME}${PROJECT_SUFFIX} --label com.docker.compose.version=$(shell docker compose version --short) --label com.docker.compose.volume=${MOUNT_VOLUME} --driver=local -o type=none -o o=bind -o "device=${DOCKER_LOCAL_DATA}/${MOUNT_FOLDER}" ${COMPOSE_PROJECT_NAME}${PROJECT_SUFFIX}_${MOUNT_VOLUME} +endif + #------------# # Production # #------------# @@ -476,6 +495,7 @@ clean_folders: clean_logs ( rm -rf html/data/i18n/ || true ) ( rm -rf html/{css,js}/dist/ || true ) ( rm -rf tmp/ || true ) + ( rm -rf build-cache/ || true ) clean_logs: ( rm -f logs/* logs/apache2/* logs/nginx/* || true )