Merge branch 'main' into aleene-week46

.github/workflows/pull_request.yml

@@ -192,7 +192,7 @@ jobs:
         echo "export USER_GID=$(id -g)" >> .envrc
     - name: Test make dev
       run: |
-        make DOCKER_LOCAL_DATA="$(pwd)" dev
+        make DOCKER_LOCAL_DATA="$(pwd)" SKIP_SAMPLE_IMAGES=1 dev
         make status
     - name: Test all is running
       run: make livecheck || ( tail -n 300 logs/apache2/*error*log; docker compose logs; false )

Makefile

@@ -11,6 +11,8 @@ SHELL := $(shell which bash)
 ENV_FILE ?= .env
 NAME = "ProductOpener"
 MOUNT_POINT ?= /mnt
+# in CI, in make dev we want to skip downloading sample images (too slow)
+SKIP_SAMPLE_IMAGES ?= SKIP_SAMPLE_IMAGES
 DOCKER_LOCAL_DATA_DEFAULT = /srv/off/docker_data
 DOCKER_LOCAL_DATA ?= $(DOCKER_LOCAL_DATA_DEFAULT)
 OS := $(shell uname)

scripts/gen_feeds_daily.sh

@@ -69,15 +69,20 @@ cd $OFF_SCRIPTS_DIR
 ./mongodb_dump.sh $OFF_PUBLIC_DATA_DIR $PRODUCT_OPENER_FLAVOR $MONGODB_HOST $PRODUCT_OPENER_FLAVOR_SHORT
 
 # Small products data and images export for Docker dev environments
-# for about 1/10000th of the products contained in production.
-./export_products_data_and_images.pl --sample-mod 10000,0 \
+# for about 1/100000th of the products contained in production.
+./export_products_data_and_images.pl --sample-mod 100000,0 \
+    --products-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.tar.gz \
+    --images-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.images.tar.gz \
+    --jsonl-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.jsonl.gz \
+    --mongo-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-100000.mongodbdump.gz
+# On saturday, export modulo 1000 and 10000 for larger sample
+if [ "$(date +%u)" = "6" ]
+then
+    ./export_products_data_and_images.pl --sample-mod 10000,0 \
     --products-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.tar.gz \
     --images-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.images.tar.gz \
     --jsonl-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.jsonl.gz \
     --mongo-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-10000.mongodbdump.gz
-# On saturday, export modulo 1000 for larger sample
-if [ "$(date +%u)" = "6" ]
-then
     ./export_products_data_and_images.pl --sample-mod 1000,0 \
         --products-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-1000.tar.gz \
         --images-file $OFF_PUBLIC_EXPORTS_DIR/products.random-modulo-1000.images.tar.gz \

scripts/import_sample_data.sh

@@ -7,14 +7,19 @@ cd /tmp
 echo "\033[32m------------------ 1/ Retrieve products -----------------\033[0m";
 # explicitly specify the wget output file name so that wget does not append .1 if already present
 # e.g. if the tar command failed and the script was stopped
-wget -O products.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-10000.tar.gz 2>&1
+wget -O products.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-100000.tar.gz 2>&1
 tar -xzvf products.tar.gz -C /mnt/podata/products
 rm products.tar.gz
 
-echo "\033[32m------------------ 2/ Retrieve product images -------------------\033[0m";
-wget -O products.images.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-10000.images.tar.gz 2>&1
-tar -xzvf products.images.tar.gz -C /opt/product-opener/html/images/products/
-rm products.images.tar.gz
+if [[ -z "${SKIP_SAMPLE_IMAGES}" ]]
+then
+    echo "\033[32m------------------ 2/ Retrieve product images -------------------\033[0m";
+    wget -O products.images.tar.gz https://static.openfoodfacts.org/exports/products.random-modulo-100000.images.tar.gz 2>&1
+    tar -xzvf products.images.tar.gz -C /opt/product-opener/html/images/products/
+    rm products.images.tar.gz
+else
+    echo "\033[32m------------------ SKIPPED product images -------------------\033[0m";
+fi
 
 echo "\033[32m------------------ 3/ Import products -------------------\033[0m";
 perl -I/opt/product-opener/lib /opt/product-opener/scripts/update_all_products_from_dir_in_mongodb.pl