From c927873e4239e64657a234c76939c60b36bf4a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Wed, 7 Aug 2024 11:42:25 +0200 Subject: [PATCH] fix: intermarche import (#10652) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixing an issue that prevented importing files from Intermarché Co-authored-by: OFF --- .../imports/intermarche/run_intermarche_import.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/imports/intermarche/run_intermarche_import.sh b/scripts/imports/intermarche/run_intermarche_import.sh index f8a3b0f1dcc81..ee0471a80503c 100755 --- a/scripts/imports/intermarche/run_intermarche_import.sh +++ b/scripts/imports/intermarche/run_intermarche_import.sh @@ -34,15 +34,18 @@ echo "IMPORT_SINCE: $IMPORT_SINCE days" # directory, so we move them to a different place, but still keep past files # in case we need to reprocess them echo "Copy csv and image files to $OFF_SFTP_HOME_DIR/artinformatique-backup/data/intermarche/" -cp -a $OFF_SFTP_HOME_DIR/artinformatique/data/intermarche/* $OFF_SFTP_HOME_DIR/artinformatique-backup/data/intermarche/ +# Note: the "cp -a" flag results in an error if the files are owned by different users +# removing it as it is making this script too fragile +# cp: preserving times for '/mnt/off-pro/sftp/artinformatique-backup/data/intermarche/23-07-2023-intermarche.csv': Operation not permitted +cp -r $OFF_SFTP_HOME_DIR/artinformatique/data/intermarche $OFF_SFTP_HOME_DIR/artinformatique-backup/data/ -# move CSV files modified since the last successful run +# copy CSV files modified since the last successful run echo "Move CSV files modified since the last successful run" rm -rf $DATA_TMP_DIR mkdir $DATA_TMP_DIR mkdir $DATA_TMP_DIR/data -find $OFF_SFTP_HOME_DIR/artinformatique/data/intermarche/ -mtime -$IMPORT_SINCE -type f -name "*.csv*" -exec mv {} $DATA_TMP_DIR/data/ \; +find $OFF_SFTP_HOME_DIR/artinformatique/data/intermarche/ -mtime -$IMPORT_SINCE -type f -name "*.csv*" -exec cp {} $DATA_TMP_DIR/data/ \; # The CSV files are in a format like 19-11-2023-intermarche.csv and sometimes they # send us files from multiple months @@ -83,5 +86,8 @@ for file in $DATA_TMP_DIR/data/*.csv; do ./scripts/import_csv_file.pl --csv_file $file --user_id perfqual --comment "Import Perfqual" --source_id "les-mousquetaires" --source_name "Les Mousquetaires" --source_url "https://www.intermarche.com/" --manufacturer --org_id les-mousquetaires --define lc=fr --images_dir $IMAGES_TMP_DIR done +# Remove the files from the sftp so that Intermarché knows that they have been successfully processed +find $OFF_SFTP_HOME_DIR/artinformatique/data/intermarche/ -mtime -$IMPORT_SINCE -type f -name "*.csv*" -exec mv {} $DATA_TMP_DIR/data/ \; + # mark successful run mark_successful_run $SUCCESS_FILE_PATH