chore: Fix producers import timer (#9486)

* fix: producers imports script
* fix: producers import timer
* chore: better deployment verification

---------

Co-authored-by: Stéphane Gigandet <[email protected]>

conf/systemd/producers_import.service → conf/systemd/producers_import@.service

@@ -5,7 +5,7 @@ OnFailure=email-failures@producers_import__%i.service
 
 [Service]
 Type=oneshot
-# service instance name "%i" is off / obf / opff / opf
+# service instance name "%i" is off-pro
 Environment=PERL5LIB=/srv/%i/lib/:/usr/local/share/perl/5.32.1/:/usr/lib/perl5
 WorkingDirectory=/srv/%i
 ExecStart=/srv/%i/scripts/imports/producers_imports.sh

scripts/deploy/verify-symlinks.sh → scripts/deploy/verify-deployment.sh

@@ -3,9 +3,29 @@
 #
 # short name of service must be provided as argument
 
-SERVICE=$1
+if [[ $(id --user) != 0 ]]
+then
+  >&2 echo "ERROR: This script must be run as root"
+fi
 
-KNOWN_SERVICES=(obf off off-pro opf opff)
+while [[ $# -gt 0 ]]
+do
+  case $1 in
+        "--verbose" | "-v")
+          VERBOSE=1
+          shift;;
+        *)
+          if [[ -n "$SERVICE" ]]
+          then
+            echo >&2 "ERROR: only one service can be verified at a time (for now):"
+            exit 11
+          fi
+          SERVICE=$1
+          shift;;
+  esac
+done
+
+KNOWN_SERVICES=(obf off opf opff)
 
 declare -A LONG_NAMES
 LONG_NAMES[obf]=openbeautyfacts
@@ -17,6 +37,10 @@ LONG_NAMES[opff]=openpetfoodfacts
 # in the sens of `ln -s target destination``
 declare -A EXPECTED_LINKS
 
+declare -a SYSTEMD_LINKS
+declare -a SYSTEMD_UNITS_ENABLED
+declare -a SYSTEMD_UNITS_ACTIVE
+
 function is_pro {
   # return -pro if it end by -pro
   echo $(expr match "$1" '.*\(-pro\)')
@@ -28,23 +52,47 @@ function non_pro_name {
 }
 
 function check_args {
-  # check service name
-  if ! ( echo $KNOWN_SERVICES | grep -w "$SERVICE" )
+  if [[ -z "$SERVICE" ]]
   then
-    >2 echo "unknown service: $SERVICE"
+    >&2 echo "ERROR: Please provide a service name as argument"
     exit -1
   fi
   REPO_PATH="/srv/$SERVICE"
   # check service is deployed as expected
-  if [[ ! -d $REPO_PATH/.git ]];
-    >2 echo "$REPO_PATH must be a git repository"
+  if [[ ! -d $REPO_PATH/.git ]]
+  then
+    >&2 echo "ERROR: $REPO_PATH must be a git repository"
     exit -2
   fi
   ZFS_PATH="/mnt/$SERVICE"
   IS_PRO=$(is_pro "$SERVICE")
   NON_PRO_SERVICE=$(non_pro_name "$SERVICE")
   PRO_SERVICE=$NON_PRO_SERVICE"-pro"
   SERVICE_LONG_NAME=${LONG_NAMES[$NON_PRO_SERVICE]}
+  # check service name
+  if ! ( echo "${KNOWN_SERVICES[@]}" | grep -w -q "$NON_PRO_SERVICE" )
+  then
+    >&2 echo "unknown service: $SERVICE"
+    exit -1
+  fi
+}
+
+function compute_services {
+  # systemd services to check for symlinks
+  SYSTEMD_LINKS+=( [email protected] nginx.service.d apache2.service.d [email protected] )
+  # units that must be active (and enabled)
+  SYSTEMD_UNITS_ACTIVE=( nginx.service apache2.service cloud_vision_ocr@$SERVICE.service )
+  SYSTEMD_UNITS_ENABLED=( )
+  if [[ -z $IS_PRO ]]
+  then
+    SYSTEMD_LINKS+=( gen_feeds{,_daily}@.{service,timer} )
+    SYSTEMD_UNITS_ACTIVE+=( gen_feeds@$SERVICE.timer gen_feeds_daily@$SERVICE.timer )
+    SYSTEMD_UNITS_ENABLED+=( gen_feeds@$SERVICE.service gen_feeds_daily@$SERVICE.service )
+  else
+    SYSTEMD_LINKS+=( producers_import@.{service,timer} )
+    SYSTEMD_UNITS_ACTIVE+=( producers_import@$SERVICE.timer )
+    SYSTEMD_UNITS_ENABLED+=( producers_import@$SERVICE.service )
+  fi
 }
 
 function compute_expected_links {
@@ -71,10 +119,13 @@ function compute_expected_links {
   EXPECTED_LINKS["$REPO_PATH/html/files"]="$ZFS_PATH/html_data/files"
 
   # .well-known
-  for path in apple-app-site-association apple-developer-merchantid-domain-association
-  do
-    EXPECTED_LINKS["$REPO_PATH/html/.well-known/$path"]="$REPO_PATH/conf/well-known/$SERVICE-$path"
-  done
+  if [[ -z $IS_PRO ]]
+  then
+    for path in apple-app-site-association apple-developer-merchantid-domain-association
+    do
+      EXPECTED_LINKS["$REPO_PATH/html/.well-known/$path"]="$REPO_PATH/conf/well-known/$SERVICE-$path"
+    done
+  fi
   # deeper link in zfs storages
   EXPECTED_LINKS["$REPO_PATH/deleted.images"]="$ZFS_PATH/deleted.images"
   EXPECTED_LINKS["$REPO_PATH/reverted_products"]="$ZFS_PATH/reverted_products"
@@ -98,7 +149,7 @@ function compute_expected_links {
   EXPECTED_LINKS["$REPO_PATH/export_files"]="$ZFS_PATH/cache/export_files"
 
   # exchange path
-  if [[ -z $IS_PRO]]
+  if [[ -z $IS_PRO ]]
   then
     EXPECTED_LINKS["/srv/$PRO_SERVICE/export_files"]="/mnt/$PRO_SERVICE/cache/export_files"
   fi
@@ -131,19 +182,18 @@ function compute_expected_links {
   EXPECTED_LINKS["/etc/nginx/snippets/off.cors-headers.include"]="$REPO_PATH/conf/nginx/snippets/off.cors-headers.include"
   EXPECTED_LINKS["/etc/nginx/conf.d/log_format_realip.conf"]="$REPO_PATH/conf/nginx/conf.d/log_format_realip.conf"
   EXPECTED_LINKS["/etc/nginx/mime.types"]="$REPO_PATH/conf/nginx/mime.types"
-  if [[ $SERVICE eq "off" ]]
+  if [[ $SERVICE = "off" ]]
   then
-    EXPECTED_LINKS["/etc/nginx/snippets/off.domain-redirects.include"]="$REPO_PATH/conf/nginx/off.domain-redirects.include"
-    EXPECTED_LINKS["/etc/nginx/snippets/off.locations-redirects.include"]="$REPO_PATH/conf/nginx/off.locations-redirects.include"
+    EXPECTED_LINKS["/etc/nginx/snippets/off.domain-redirects.include"]="$REPO_PATH/conf/nginx/snippets/off.domain-redirects.include"
+    EXPECTED_LINKS["/etc/nginx/snippets/off.locations-redirects.include"]="$REPO_PATH/conf/nginx/snippets/off.locations-redirects.include"
   fi
 
   # apache2 links
   EXPECTED_LINKS["/etc/apache2/ports.conf"]="$REPO_PATH/conf/apache-2.4/$SERVICE-ports.conf"
   EXPECTED_LINKS["/etc/apache2/mods-available/mpm_prefork.conf"]="$REPO_PATH/conf/apache-2.4/$SERVICE-mpm_prefork.conf"
-  EXPECTED_LINKS["/etc/apache2/sites-enabled/$SERVICE"]="$REPO_PATH/conf/apache-2.4/sites-available/$SERVICE"
+  EXPECTED_LINKS["/etc/apache2/sites-enabled/$SERVICE.conf"]="$REPO_PATH/conf/apache-2.4/sites-available/$SERVICE.conf"
 
-  # systemd links
-  for systemd_unit in {apache2,nginx}.service.d [email protected] gen_feeds_{,daily}@.{service,timer}
+  for systemd_unit in {apache2,nginx}.service.d ${SYSTEMD_LINKS[@]}
   do
     EXPECTED_LINKS["/etc/systemd/system/$systemd_unit"]="$REPO_PATH/conf/systemd/$systemd_unit"
   done
@@ -159,36 +209,78 @@ function compute_expected_links {
 
 # check links
 function check_links {
+  [[ -n "$VERBOSE" ]] && echo "Checking links..."
   for target in ${!EXPECTED_LINKS[@]}
   do
     destination=${EXPECTED_LINKS[$target]}
-    if [[ $destination -ef $target ]]
+    if [[ ! $(readlink -f $destination) = $(readlink -f $target) ]]
     then
       GOT_ERROR=1
-      >2 echo "WRONG link: $target -> $destination"
+      if [[ ! -e $target ]]
+      then
+        >&2 echo "ERROR: link $target does not exist"
+      else
+        if [[ ! -e $destination ]]
+        then
+          >&2 echo "ERROR: link $destination does not exist"
+        else
+          >&2 echo "ERROR: link instead of $target -> $destination, got $(readlink -f $target) instead"
+        fi
+      fi
+    else
+      [[ -n "$VERBOSE" ]] && echo "    OK link: $target -> $destination"
+    fi
+  done
+}
+
+
+function check_systemd_units {
+  for unit in ${SYSTEMD_UNITS_ENABLED[@]}
+  do
+    if ! ( systemctl -q is-enabled $unit )
+    then
+      >&2 echo "ERROR: $unit unit must be enabled"
+    else
+      [[ -n "$VERBOSE" ]] && echo "    OK: $unit unit enabled"
     fi
   done
+  for unit in ${SYSTEMD_UNITS_ACTIVE[@]}
+  do
+    if ! ( systemctl -q is-active $unit )
+    then
+      >&2 echo "ERROR: $unit unit must be enabled"
+    else
+      [[ -n "$VERBOSE" ]] && echo "    OK: $unit unit enabled"
+    fi
+  done
+
 }
 
 
 function other_checks {
   # apache2 must run with off user and group
   for variable in USER GROUP
-  do 
-    if ! ( grep "^export APACHE_RUN_$variable=off" /etc/apache2/envvars )
+  do
+    if ! ( grep -q "^export APACHE_RUN_$variable=off" /etc/apache2/envvars )
     then
       GOT_ERROR=1
-      >2 echo "WRONG $variable for apache2: it should be off instead of" $(grep "^export APACHE_RUN_$variable=.*" /etc/apache2/envvars)
+      >&2 echo "ERROR: $variable for apache2 should be off instead off" $(grep "^export APACHE_RUN_$variable=.*" /etc/apache2/envvars)
+    else
+      [[ -n "$VERBOSE" ]] && echo "    OK APACHE_RUN_$variable for apache2"
     fi
   done
 }
 
 
 # run
 check_args
+# must be before compute_expected_links
+compute_services
 compute_expected_links
 GOT_ERROR=0
 check_links
+check_systemd_units
+other_checks
 if [[ $GOT_ERROR -ne 0 ]]; then
   exit -3
-fi
+fi

scripts/imports/imports_utils.sh

@@ -4,9 +4,9 @@
 
 # find last run and deduce how many days to fetch
 #
-# first arg should be the path to the import success file 
+# first arg should be the path to the import success file
 # containing a timestamp of last successful import
-# it may not exists yet, in which case we default to one week
+# it may not exists yet, in which case we default to one year
 function import_since() {
     SUCCESS_FILE_PATH=$1
     if [[ -z "$SUCCESS_FILE_PATH" ]]

scripts/imports/producers_imports.sh

@@ -1,11 +1,19 @@
 #!/usr/bin/env bash
+export PERL5LIB="lib:${PERL5LIB}"
 
-# Equadis import
-./scripts/imports/equadis/run_equadis_import.sh
-# Agena import
-./scripts/imports/agena3000/run_agena3000_import.sh
-# Carrefour
-./scripts/imports/carrefour/import_carrefour.sh
+# run import scripts
+PRODUCERS=(
+    equadis
+    agena3000
+    carrefour
+    intermarche
+)
+for PRODUCER in "${PRODUCERS[@]}"
+do
+  script_path="scripts/imports/${PRODUCER}/run_${PRODUCER}_import.sh"
+  [[ -e $script_path ]] || >&2 echo "No script found for $PRODUCER"
+  ./$script_path
+done
 
-# Export
+# export
 ./scripts/export_producers_platform_data_to_public_database.sh