diff --git a/.gitlab/ci/.gitlab-ci.bucket-upload.yml b/.gitlab/ci/.gitlab-ci.bucket-upload.yml index 413fdbc5d9ef..60f3a8b4656e 100644 --- a/.gitlab/ci/.gitlab-ci.bucket-upload.yml +++ b/.gitlab/ci/.gitlab-ci.bucket-upload.yml @@ -232,6 +232,7 @@ install-packs-in-server-master: PRODUCT_TYPE: "XSIAM" SERVER_TYPE: "XSIAM" ARTIFACTS_FOLDER: "${ARTIFACTS_FOLDER_MPV2}" + ARTIFACTS_FOLDER_INSTANCE: "${ARTIFACTS_FOLDER_MPV2}/instance_${INSTANCE_ROLE}" ARTIFACTS_FOLDER_SERVER_TYPE: "${ARTIFACTS_FOLDER_MPV2}/server_type_${SERVER_TYPE}" ENV_RESULTS_PATH: "${ARTIFACTS_FOLDER_SERVER_TYPE}/env_results.json" INSTANCE_CREATED: "true" @@ -264,9 +265,7 @@ install-packs-in-server-master: else echo "${CLOUD_API_KEYS}" > "cloud_api_keys.json" fi - - section_start "Clean Machine" - - ./Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh || EXIT_CODE=$? - - section_end "Clean Machine" + - !reference [.uninstall-packs-and-reset-bucket-cloud] - section_start "Run end to end sanity tests" - ./Tests/scripts/run_end_to_end_tests.sh || EXIT_CODE=$? diff --git a/.gitlab/ci/.gitlab-ci.build-machines-cleanup.yml b/.gitlab/ci/.gitlab-ci.build-machines-cleanup.yml index df91a2815538..f609e8446005 100644 --- a/.gitlab/ci/.gitlab-ci.build-machines-cleanup.yml +++ b/.gitlab/ci/.gitlab-ci.build-machines-cleanup.yml @@ -39,6 +39,7 @@ build-machines-cleanup: CLOUD_API_KEYS: $XSIAM_API_KEYS CLOUD_API_TOKENS: $XSIAM_TOKENS ARTIFACTS_FOLDER: "${ARTIFACTS_FOLDER_MPV2}" + ARTIFACTS_FOLDER_INSTANCE: "${ARTIFACTS_FOLDER_MPV2}/instance_${INSTANCE_ROLE}" ARTIFACTS_FOLDER_SERVER_TYPE: "${ARTIFACTS_FOLDER_MPV2}/server_type_${SERVER_TYPE}" ENV_RESULTS_PATH: "${ARTIFACTS_FOLDER_SERVER_TYPE}/env_results.json" GCS_MARKET_BUCKET: "${GCS_MARKET_V2_BUCKET}" diff --git a/Tests/scripts/prepare_content_packs_for_testing.sh b/Tests/scripts/prepare_content_packs_for_testing.sh index e64277c05c0b..101e73aaf816 100644 --- a/Tests/scripts/prepare_content_packs_for_testing.sh +++ b/Tests/scripts/prepare_content_packs_for_testing.sh @@ -32,7 +32,7 @@ fi # We can freely use these buckets since its only reading the prod to the circle-ci bucket. echo "Preparing content packs for testing ..." -gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER}/logs/gcloud_auth.log" 2>&1 +gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER_SERVER_TYPE}/logs/gcloud_auth.log" 2>&1 echo "Auth loaded successfully." # ====== BUILD CONFIGURATION ====== @@ -60,7 +60,7 @@ else SOURCE_PATH="upload-flow/builds/$CI_COMMIT_BRANCH/$CI_PIPELINE_ID/content" # ====== UPDATING TESTING BUCKET ====== echo "Copying production bucket files at: gs://$GCS_PRODUCTION_BUCKET/content to testing bucket at path: gs://$GCS_MARKET_BUCKET/$SOURCE_PATH ..." - gsutil -m cp -r "gs://$GCS_PRODUCTION_BUCKET/content" "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" > "$ARTIFACTS_FOLDER/logs/Prepare Content Packs For Testing gsutil.log" 2>&1 + gsutil -m cp -r "gs://$GCS_PRODUCTION_BUCKET/content" "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" >> "${ARTIFACTS_FOLDER_SERVER_TYPE}/logs/Prepare_Content_Packs_For_Testing_gsutil.log" 2>&1 echo "Finished copying successfully." # ====== UPDATING TESTING BUCKET ====== else # case 3: a test upload, the source is of an exiting target bucket, no need to copy from the prod bucket @@ -71,7 +71,7 @@ else fi echo "Copying master files at: gs://$GCS_MARKET_BUCKET/$SOURCE_PATH to target path: gs://$BUILD_BUCKET_CONTENT_DIR_FULL_PATH ..." -gsutil -m cp -r "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" "gs://$BUILD_BUCKET_CONTENT_DIR_FULL_PATH" > "$ARTIFACTS_FOLDER/logs/Prepare Content Packs For Testing gsutil.log" 2>&1 +gsutil -m cp -r "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" "gs://$BUILD_BUCKET_CONTENT_DIR_FULL_PATH" >> "${ARTIFACTS_FOLDER_SERVER_TYPE}/logs/Prepare_Content_Packs_For_Testing_gsutil.log" 2>&1 echo "Finished copying successfully." CONTENT_PACKS_TO_UPLOAD_FILE="${ARTIFACTS_FOLDER_SERVER_TYPE}/content_packs_to_upload.txt" @@ -119,25 +119,25 @@ else fi python3 ./Tests/Marketplace/upload_packs.py -pa "${PACK_ARTIFACTS}" -d "${ARTIFACTS_FOLDER_SERVER_TYPE}/packs_dependencies.json" --artifacts-folder-server-type "${ARTIFACTS_FOLDER_SERVER_TYPE}" -e $EXTRACT_FOLDER -b $GCS_BUILD_BUCKET -s "$GCS_MARKET_KEY" -n $CI_PIPELINE_ID -pn "${CONTENT_PACKS_TO_UPLOAD}" -p $UPLOAD_SPECIFIC_PACKS -o $OVERRIDE_ALL_PACKS -sb $BUILD_BUCKET_PACKS_DIR_PATH -k $PACK_SIGNING_KEY -rt $REMOVE_PBS -bu $BUCKET_UPLOAD_FLOW -pb "$GCS_PRIVATE_BUCKET" -c $CI_COMMIT_BRANCH -f $IS_FORCE_UPLOAD -dz "$CREATE_DEPENDENCIES_ZIP" -mp "$MARKETPLACE_TYPE" - if [ -f "$ARTIFACTS_FOLDER/index.json" ]; then - gsutil cp -z json "$ARTIFACTS_FOLDER/index.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" + if [ -f "${ARTIFACTS_FOLDER_SERVER_TYPE}/index.json" ]; then + gsutil cp -z json "${ARTIFACTS_FOLDER_SERVER_TYPE}/index.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" else - echo "Skipping uploading index.json file." + echo "Skipping uploading ${ARTIFACTS_FOLDER_SERVER_TYPE}/index.json file, it doesn't exist." fi - corepacks_files_count=$(find $ARTIFACTS_FOLDER -name "corepacks*.json" | wc -l) - if [ $corepacks_files_count -eq 0 ]; then - echo "No corepacks files were found, skipping uploading." + core_packs_files_count=$(find "${ARTIFACTS_FOLDER_SERVER_TYPE}" -name "corepacks*.json" | wc -l) + if [ "${core_packs_files_count}" -eq 0 ]; then + echo "No core packs files were found, skipping uploading." else - echo "Uploading corepacks files." - # Copy corepacks files from the artifacts folder to the build bucket: - find $ARTIFACTS_FOLDER -name "corepacks*.json" -exec gsutil cp -z json {} "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" \; - echo "Successfully uploaded corepacks files." + echo "Uploading ${core_packs_files_count} core packs files." + # Copy core packs files from the artifacts folder to the build bucket: + find "${ARTIFACTS_FOLDER_SERVER_TYPE}" -name "corepacks*.json" -exec gsutil cp -z json "{}" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" \; + echo "Successfully uploaded core packs files." fi - if [ -f "$ARTIFACTS_FOLDER/versions-metadata.json" ]; then + if [ -f "${ARTIFACTS_FOLDER_SERVER_TYPE}/versions-metadata.json" ]; then echo "Uploading versions-metadata.json." - gsutil cp -z json "$ARTIFACTS_FOLDER/versions-metadata.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" + gsutil cp -z json "${ARTIFACTS_FOLDER_SERVER_TYPE}/versions-metadata.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" echo "Successfully uploaded versions-metadata.json." else echo "No versions-metadata.json file, skipping uploading." diff --git a/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh b/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh index a7f909ed665e..5e7d1c0ab7ae 100644 --- a/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh +++ b/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh @@ -25,13 +25,13 @@ fi if [[ -z "${CLOUD_CHOSEN_MACHINE_IDS}" ]]; then exit_on_error 1 "CLOUD_CHOSEN_MACHINE_IDS is not defined" else - gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER}/logs/gcloud_auth.log" 2>&1 + gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER_INSTANCE}/logs/gcloud_auth.log" 2>&1 exit_on_error $? "Failed to authenticate with GCS_MARKET_KEY" IFS=', ' read -r -a CLOUD_CHOSEN_MACHINE_ID_ARRAY <<< "${CLOUD_CHOSEN_MACHINE_IDS}" for CLOUD_CHOSEN_MACHINE_ID in "${CLOUD_CHOSEN_MACHINE_ID_ARRAY[@]}"; do echo "Copying prod bucket to ${CLOUD_CHOSEN_MACHINE_ID} bucket." - gsutil -m cp -r "gs://$GCS_SOURCE_BUCKET/content" "gs://$GCS_MACHINES_BUCKET/${CLOUD_CHOSEN_MACHINE_ID}/" > "$ARTIFACTS_FOLDER/Copy_prod_bucket_to_cloud_machine_cleanup.log" 2>&1 + gsutil -m cp -r "gs://$GCS_SOURCE_BUCKET/content" "gs://$GCS_MACHINES_BUCKET/${CLOUD_CHOSEN_MACHINE_ID}/" >> "${ARTIFACTS_FOLDER_INSTANCE}/logs/Copy_prod_bucket_to_cloud_machine_cleanup.log" 2>&1 exit_on_error $? "Failed to copy prod bucket to ${CLOUD_CHOSEN_MACHINE_ID} bucket" done