Merge pull request #332 from pagopa/develop

feat: Promote report feature to master

acceptance_tests/common/setup.sh

@@ -4,6 +4,7 @@ mkdir -p workdir/hpans
 mkdir -p workdir/ade-errors
 mkdir -p workdir/output/pending
 mkdir -p workdir/logs
+mkdir -p workdir/reports
 
 git clone --depth 1 https://github.com/pagopa/cstar-cli.git
 cd cstar-cli || exit

acceptance_tests/file_report/file_report.bash

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+if [ $# -lt 1 ] ; then
+    echo "Illegal number of parameters (1 mandatory, was $#)" >&1
+    echo "usage: bash script_splitting.bash env [timeout_in_minutes]" >&1
+    exit 2
+fi
+
+if [ "$1" != "uat" ] &&  [ "$1" != "dev" ]
+then
+    echo "only dev and uat available for this test!"
+    exit 2
+fi
+
+ENV=$1
+# timeout default 5minutes
+TIMEOUT_IN_MINUTES="${2:-5}"
+
+sh ../common/setup.sh
+
+N_AGGREGATES=8
+
+generate_input_file() {
+    cd cstar-cli || exit
+    echo "Generating input file..."
+    poetry run cst sender aggregates --sender 12345 --action trx_and_aggr --aggr-qty $N_AGGREGATES --avg-trx 3
+    cd ..
+    FILENAME=$(basename cstar-cli/generated/*.csv)
+    echo "Generated: $FILENAME"
+    cp cstar-cli/generated/"$FILENAME" ./workdir/input/"$FILENAME"
+}
+run_batch_service() {
+    java -jar ../common/rtd-ms-transaction-filter.jar
+}
+get_file_sent_occurrence_in_report() {
+    OUTPUT_FILENAME=$(basename workdir/output/ADE.*)
+    FILE_REPORT_NAME=$(ls -v workdir/reports | tail -n 1)
+    FILE_SENT_OCCURRANCE_IN_REPORT=$(grep -c "$OUTPUT_FILENAME" < workdir/reports/"$FILE_REPORT_NAME")
+}
+
+### file generation
+generate_input_file
+
+### batch service configuration
+# shellcheck source=../common/setenv_env.sh
+source ../common/setenv_"$ENV".sh
+source setenv.sh
+
+echo "Executing batch service..."
+### batch service run
+run_batch_service
+
+#### ASSERTIONS
+
+# the file sent is not in the report
+get_file_sent_occurrence_in_report
+if [ "$FILE_SENT_OCCURRANCE_IN_REPORT" != 0 ]
+then
+  echo "File sent has been found in report but it was not supposed to: [FAILED]"
+  exit 2
+fi
+
+# check if file has been uploaded
+N_UPLOADS=$(grep -c "uploaded with success (status was: 201)" < workdir/logs/application.log)
+if [ "$N_UPLOADS" -ne 1 ]
+then
+	echo "Upload test not passed, $N_UPLOADS files uploaded: [FAILED]"
+	exit 2
+else
+	echo "Files uploaded with success: [SUCCESS]"
+fi
+
+SLEEP_INTERVAL_IN_SECONDS=10
+
+#set batch service send to false in order to not send the placeholder files
+export ACQ_BATCH_TRX_SENDER_ADE_ENABLED=false
+for (( i=0 ; i <= TIMEOUT_IN_MINUTES * 60 / SLEEP_INTERVAL_IN_SECONDS; i++))
+do
+    echo "Waiting $SLEEP_INTERVAL_IN_SECONDS seconds..."
+    sleep $SLEEP_INTERVAL_IN_SECONDS
+
+    # run batch service with dummy file
+    cp cstar-cli/generated/"$FILENAME" ./workdir/input/"$FILENAME"
+    run_batch_service
+
+    # check if file sent in the previous run has been received inside the report
+    get_file_sent_occurrence_in_report
+
+    # if report does contain the file sent the exit loop
+    if [ "$FILE_SENT_OCCURRANCE_IN_REPORT" -gt 0 ]
+    then
+        echo "file found in report: [SUCCESS]"
+        break
+    fi
+
+done
+
+exit 0

acceptance_tests/file_report/file_report_more_sender_codes.bash

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+if [ $# -lt 1 ] ; then
+    echo "Illegal number of parameters (1 mandatory, was $#)" >&1
+    echo "usage: bash script_splitting.bash env [timeout_in_minutes] [sender_code_1] [sender_code_2]" >&1
+    exit 2
+fi
+
+if [ "$1" != "uat" ] &&  [ "$1" != "dev" ]
+then
+    echo "only dev and uat available for this test!"
+    exit 2
+fi
+
+
+ENV=$1
+# timeout default 5minutes
+TIMEOUT_IN_MINUTES="${2:-5}"
+
+# Set sender codes
+SENDER_CODE1="${3-12345}"
+SENDER_CODE2="${4-54321}"
+
+
+N_AGGREGATES=8
+
+REPORT_DIR=workdir/reports
+
+sh ../common/setup.sh
+
+echo "Make sure to have $SENDER_CODE1 and $SENDER_CODE2 associated with your API key"
+echo "Make sure to have empty cstar-cli/generated folder"
+
+generate_input_file() {
+    cd cstar-cli || exit
+    echo "Generating input file..."
+    poetry run cst sender aggregates --sender $SENDER_CODE1 --action trx_and_aggr --aggr-qty $N_AGGREGATES --avg-trx 3
+    poetry run cst sender aggregates --sender $SENDER_CODE2 --action trx_and_aggr --aggr-qty $N_AGGREGATES --avg-trx 3
+    cd ..
+    FILENAME=$(ls -1 cstar-cli/generated/*.csv | xargs -I {} basename {})
+    echo "Generated:"
+    echo "$FILENAME"
+    cp cstar-cli/generated/*.csv ./workdir/input/
+}
+run_batch_service() {
+    java -jar ../common/rtd-ms-transaction-filter.jar
+}
+get_file_sent_occurrence_in_report() {
+    OUTPUT_FILENAME=$(ls workdir/output | grep -E "ADE.*.pgp")
+    FILE_REPORT_NAME=$(ls -v "$REPORT_DIR" | tail -n 1)
+    FILES_SENT_OCCURRENCES_IN_REPORT=$(grep -c "$OUTPUT_FILENAME" < "$REPORT_DIR"/"$FILE_REPORT_NAME")
+}
+check_two_files_sent() {
+  # check if the files has been uploaded
+  N_UPLOADS=$(grep -c "uploaded with success (status was: 201)" < workdir/logs/application.log)
+  if [ "$N_UPLOADS" -ne 2 ]
+  then
+    echo "Upload test not passed, $N_UPLOADS files uploaded: [FAILED]"
+    exit 2
+  else
+    echo "Files uploaded with success: [SUCCESS]"
+  fi
+}
+
+### file generation
+generate_input_file
+
+### batch service configuration
+# shellcheck source=../common/setenv_env.sh
+source ../common/setenv_"$ENV".sh
+source setenv.sh
+
+# Disable batch scheduling for processing both files independently
+export ACQ_BATCH_SCHEDULED=false
+
+# Explicitly set the report directory
+export ACQ_BATCH_FILE_REPORT_PATH="$REPORT_DIR"
+
+# Batch service run for both files
+echo "Executing batch service..."
+run_batch_service
+run_batch_service
+
+#### ASSERTIONS
+check_two_files_sent
+
+SLEEP_INTERVAL_IN_SECONDS=10
+
+# Set batch service send to false in order to not send the placeholder files
+export ACQ_BATCH_TRX_SENDER_ADE_ENABLED=false
+
+# Wait for the report to be returned containing the two files
+for (( i=0 ; i <= TIMEOUT_IN_MINUTES * 60 / SLEEP_INTERVAL_IN_SECONDS; i++))
+do
+    echo "Waiting $SLEEP_INTERVAL_IN_SECONDS seconds..."
+    sleep $SLEEP_INTERVAL_IN_SECONDS
+
+    # run batch service with dummy file
+    cp cstar-cli/generated/*.csv ./workdir/input/
+    run_batch_service
+
+    # check if file sent in the previous run has been received inside the report
+    get_file_sent_occurrence_in_report
+
+    # if report does contain the files sent the exit loop
+    if [ "$FILES_SENT_OCCURRENCES_IN_REPORT" -eq 2 ]
+    then
+        echo "Files found in report: [SUCCESS]"
+        break
+    fi
+
+done
+
+rm -rf cstar-cli
+rm -rf workdir
+
+exit 0

acceptance_tests/file_report/file_report_without_folder.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+
+if [ $# -lt 1 ] ; then
+    echo "Illegal number of parameters (1 mandatory, was $#)" >&1
+    echo "usage: bash script_splitting.bash env" >&1
+    exit 2
+fi
+
+if [ "$1" != "uat" ] &&  [ "$1" != "dev" ]
+then
+    echo "only dev and uat available for this test!"
+    exit 2
+fi
+
+ENV=$1
+
+N_AGGREGATES=8
+
+REPORT_DIR=workdir/reports
+
+sh ../common/setup.sh
+
+rm -rf "$REPORT_DIR"
+
+generate_input_file() {
+    cd cstar-cli || exit
+    echo "Generating input file..."
+    poetry run cst sender aggregates --sender 12345 --action trx_and_aggr --aggr-qty $N_AGGREGATES --avg-trx 3
+    cd ..
+    FILENAME=$(basename cstar-cli/generated/*.csv)
+    echo "Generated: $FILENAME"
+    cp cstar-cli/generated/"$FILENAME" ./workdir/input/"$FILENAME"
+}
+run_batch_service() {
+    java -jar ../common/rtd-ms-transaction-filter.jar
+}
+get_file_sent_occurrence_in_report() {
+     OUTPUT_FILENAME=$(ls workdir/output | grep -E "ADE.*.pgp")
+     FILE_REPORT_NAME=$(ls -v "$REPORT_DIR" | tail -n 1)
+     FILE_SENT_OCCURRENCE_IN_REPORT=$(grep -c $OUTPUT_FILENAME < "$REPORT_DIR"/"$FILE_REPORT_NAME")
+ }
+
+### file generation
+generate_input_file
+
+### batch service configuration
+# shellcheck source=../common/setenv_env.sh
+source ../common/setenv_"$ENV".sh
+source setenv.sh
+
+# force reports directory to be workdir/reports
+export ACQ_BATCH_FILE_REPORT_PATH="$REPORT_DIR"
+export ACQ_BATCH_TRX_SENDER_ADE_ENABLED=false
+
+echo "Executing batch service..."
+### batch service run
+run_batch_service
+#### ASSERTIONS
+
+# expected result: job went fine and report directory has been created
+if [ -d "$ACQ_BATCH_FILE_REPORT_PATH" ]
+then
+  echo "Directory "$REPORT_DIR" does exists: [SUCCESS]"
+else
+  echo "Directory "$REPORT_DIR" does not exists: [FAILED]"
+  exit 2
+fi
+
+# one report is downloaded
+if [ "$(ls -1 "$REPORT_DIR" | wc -l)" -eq 1 ]
+then
+  echo "Directory "$REPORT_DIR" is not empty: [SUCCESS]"
+else
+  echo "Directory "$REPORT_DIR" is empty: [FAILED]"
+  exit 2
+fi
+
+rm -rf cstar-cli
+rm -rf workdir
+
+exit 0

acceptance_tests/file_report/setenv.sh

@@ -0,0 +1,6 @@
+export ACQ_BATCH_TRX_SENDER_RTD_ENABLED=false
+export ACQ_BATCH_TRX_SENDER_ADE_ENABLED=true
+
+export ACQ_BATCH_DELETE_OUTPUT_FILE=KEEP
+export ACQ_BATCH_WRITER_ADE_SPLIT_THRESHOLD=10
+export ACQ_BATCH_INPUT_CHUNK_SIZE=10

acceptance_tests/filename_uniqueness/fail_upload_file_name_not_unique.bash

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ $# -ne 1 ]; then
+  echo "Illegal number of parameters (1 mandatory, was $#)" >&1
+  echo "usage: bash script_splitting.bash env" >&1
+  exit 2
+fi
+
+ENV=$1
+
+N_AGGREGATES=10
+
+RETRY_MAX_ATTEMPTS=3
+
+sh ../common/setup.sh
+
+generate_input_file() {
+  cd cstar-cli || exit
+  echo "Generating input file..."
+  poetry run cst sender aggregates --sender 12345 --action trx_and_aggr --aggr-qty $N_AGGREGATES --avg-trx 3
+  cd ..
+  FILENAME=$(basename cstar-cli/generated/*.csv)
+  echo "Generated: $FILENAME"
+  cp cstar-cli/generated/"$FILENAME" ./workdir/input/"$FILENAME"
+}
+run_batch_service() {
+  echo "Executing batch service..."
+  java -jar ../common/rtd-ms-transaction-filter.jar
+}
+check_one_file_sent() {
+  # check if the first file has been uploaded
+  N_UPLOADS=$(grep -c "uploaded with success (status was: 201)" <workdir/logs/application.log)
+  if [ "$N_UPLOADS" -ne 1 ]; then
+    echo -e "Upload test not passed, $N_UPLOADS files uploaded: \033[1m\033[31m[FAILED]\033[0m"
+    exit 2
+  else
+    echo -e "File uploaded with success: \033[1m\033[32m[SUCCESS]\033[0m"
+  fi
+}
+check_one_file_not_uploaded_due_to_conflict() {
+  # check if the second file is not uploaded because has the same name of the first
+  N_UPLOADS=$(grep -c "(status was: 409)" <workdir/logs/application.log)
+  if [ "$N_UPLOADS" -ne "$RETRY_MAX_ATTEMPTS" ]; then
+    echo -e "Test not passed, 409 for the second file not returned: \033[1m\033[31m[FAILED]\033[0m"
+    exit 2
+  else
+    echo -e "Test passed, 409 for the second file returned: \033[1m\033[32m[SUCCESS]\033[0m"
+  fi
+}
+
+### file generation
+generate_input_file
+
+### batch service configuration
+source ../common/setenv_"$ENV".sh
+source setenv.sh
+
+### batch service run
+run_batch_service
+
+# Assertion 1. check one file uploaded correctly
+check_one_file_sent
+
+# Second Batch Service run with the same file
+cp cstar-cli/generated/"$FILENAME" ./workdir/input/"$FILENAME"
+
+run_batch_service
+
+# Assertion 2. check application log to find evidence of failed files upload
+check_one_file_not_uploaded_due_to_conflict
+
+rm -rf ./cstar-cli
+rm -rf ./workdir
+
+exit 0

acceptance_tests/filename_uniqueness/setenv.sh

@@ -0,0 +1,6 @@
+export ACQ_BATCH_TRX_SENDER_RTD_ENABLED=false
+export ACQ_BATCH_TRX_SENDER_ADE_ENABLED=true
+
+export ACQ_BATCH_DELETE_OUTPUT_FILE=KEEP
+export ACQ_BATCH_WRITER_ADE_SPLIT_THRESHOLD=10
+export ACQ_BATCH_INPUT_CHUNK_SIZE=10