fix prepare_content_packs_for_testing.sh (#30749)

fix prepare_content_packs_for_testing.sh #30749
demisto · Nov 8, 2023 · f29ddb3 · f29ddb3
1 parent 343cd74
commit f29ddb3
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 20 deletions.
diff --git a/.gitlab/ci/.gitlab-ci.bucket-upload.yml b/.gitlab/ci/.gitlab-ci.bucket-upload.yml
@@ -232,6 +232,7 @@ install-packs-in-server-master:
     PRODUCT_TYPE: "XSIAM"
     SERVER_TYPE: "XSIAM"
     ARTIFACTS_FOLDER: "${ARTIFACTS_FOLDER_MPV2}"
+    ARTIFACTS_FOLDER_INSTANCE: "${ARTIFACTS_FOLDER_MPV2}/instance_${INSTANCE_ROLE}"
     ARTIFACTS_FOLDER_SERVER_TYPE: "${ARTIFACTS_FOLDER_MPV2}/server_type_${SERVER_TYPE}"
     ENV_RESULTS_PATH: "${ARTIFACTS_FOLDER_SERVER_TYPE}/env_results.json"
     INSTANCE_CREATED: "true"
@@ -264,9 +265,7 @@ install-packs-in-server-master:
       else
         echo "${CLOUD_API_KEYS}" > "cloud_api_keys.json"
       fi
-    - section_start "Clean Machine"
-    - ./Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh || EXIT_CODE=$?
-    - section_end "Clean Machine"
+    - !reference [.uninstall-packs-and-reset-bucket-cloud]
 
     - section_start "Run end to end sanity tests"
     - ./Tests/scripts/run_end_to_end_tests.sh || EXIT_CODE=$?

diff --git a/.gitlab/ci/.gitlab-ci.build-machines-cleanup.yml b/.gitlab/ci/.gitlab-ci.build-machines-cleanup.yml
@@ -39,6 +39,7 @@ build-machines-cleanup:
     CLOUD_API_KEYS: $XSIAM_API_KEYS
     CLOUD_API_TOKENS: $XSIAM_TOKENS
     ARTIFACTS_FOLDER: "${ARTIFACTS_FOLDER_MPV2}"
+    ARTIFACTS_FOLDER_INSTANCE: "${ARTIFACTS_FOLDER_MPV2}/instance_${INSTANCE_ROLE}"
     ARTIFACTS_FOLDER_SERVER_TYPE: "${ARTIFACTS_FOLDER_MPV2}/server_type_${SERVER_TYPE}"
     ENV_RESULTS_PATH: "${ARTIFACTS_FOLDER_SERVER_TYPE}/env_results.json"
     GCS_MARKET_BUCKET: "${GCS_MARKET_V2_BUCKET}"

diff --git a/Tests/scripts/prepare_content_packs_for_testing.sh b/Tests/scripts/prepare_content_packs_for_testing.sh
@@ -32,7 +32,7 @@ fi
 # We can freely use these buckets since its only reading the prod to the circle-ci bucket.
 
 echo "Preparing content packs for testing ..."
-gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER}/logs/gcloud_auth.log" 2>&1
+gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER_SERVER_TYPE}/logs/gcloud_auth.log" 2>&1
 echo "Auth loaded successfully."
 
 # ====== BUILD CONFIGURATION ======
@@ -60,7 +60,7 @@ else
     SOURCE_PATH="upload-flow/builds/$CI_COMMIT_BRANCH/$CI_PIPELINE_ID/content"
     # ====== UPDATING TESTING BUCKET ======
     echo "Copying production bucket files at: gs://$GCS_PRODUCTION_BUCKET/content to testing bucket at path: gs://$GCS_MARKET_BUCKET/$SOURCE_PATH ..."
-    gsutil -m cp -r "gs://$GCS_PRODUCTION_BUCKET/content" "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" > "$ARTIFACTS_FOLDER/logs/Prepare Content Packs For Testing gsutil.log" 2>&1
+    gsutil -m cp -r "gs://$GCS_PRODUCTION_BUCKET/content" "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" >> "${ARTIFACTS_FOLDER_SERVER_TYPE}/logs/Prepare_Content_Packs_For_Testing_gsutil.log" 2>&1
     echo "Finished copying successfully."
     # ====== UPDATING TESTING BUCKET ======
   else  # case 3: a test upload, the source is of an exiting target bucket, no need to copy from the prod bucket
@@ -71,7 +71,7 @@ else
 fi
 
 echo "Copying master files at: gs://$GCS_MARKET_BUCKET/$SOURCE_PATH to target path: gs://$BUILD_BUCKET_CONTENT_DIR_FULL_PATH ..."
-gsutil -m cp -r "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" "gs://$BUILD_BUCKET_CONTENT_DIR_FULL_PATH" > "$ARTIFACTS_FOLDER/logs/Prepare Content Packs For Testing gsutil.log" 2>&1
+gsutil -m cp -r "gs://$GCS_MARKET_BUCKET/$SOURCE_PATH" "gs://$BUILD_BUCKET_CONTENT_DIR_FULL_PATH" >> "${ARTIFACTS_FOLDER_SERVER_TYPE}/logs/Prepare_Content_Packs_For_Testing_gsutil.log" 2>&1
 echo "Finished copying successfully."
 
 CONTENT_PACKS_TO_UPLOAD_FILE="${ARTIFACTS_FOLDER_SERVER_TYPE}/content_packs_to_upload.txt"
@@ -119,25 +119,25 @@ else
   fi
   python3 ./Tests/Marketplace/upload_packs.py -pa "${PACK_ARTIFACTS}" -d "${ARTIFACTS_FOLDER_SERVER_TYPE}/packs_dependencies.json" --artifacts-folder-server-type "${ARTIFACTS_FOLDER_SERVER_TYPE}" -e $EXTRACT_FOLDER -b $GCS_BUILD_BUCKET -s "$GCS_MARKET_KEY" -n $CI_PIPELINE_ID -pn "${CONTENT_PACKS_TO_UPLOAD}" -p $UPLOAD_SPECIFIC_PACKS -o $OVERRIDE_ALL_PACKS -sb $BUILD_BUCKET_PACKS_DIR_PATH -k $PACK_SIGNING_KEY -rt $REMOVE_PBS -bu $BUCKET_UPLOAD_FLOW -pb "$GCS_PRIVATE_BUCKET" -c $CI_COMMIT_BRANCH -f $IS_FORCE_UPLOAD -dz "$CREATE_DEPENDENCIES_ZIP" -mp "$MARKETPLACE_TYPE"
 
-  if [ -f "$ARTIFACTS_FOLDER/index.json" ]; then
-    gsutil cp -z json "$ARTIFACTS_FOLDER/index.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH"
+  if [ -f "${ARTIFACTS_FOLDER_SERVER_TYPE}/index.json" ]; then
+    gsutil cp -z json "${ARTIFACTS_FOLDER_SERVER_TYPE}/index.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH"
   else
-    echo "Skipping uploading index.json file."
+    echo "Skipping uploading ${ARTIFACTS_FOLDER_SERVER_TYPE}/index.json file, it doesn't exist."
   fi
 
-  corepacks_files_count=$(find $ARTIFACTS_FOLDER -name "corepacks*.json" | wc -l)
-  if [ $corepacks_files_count -eq 0 ]; then
-    echo "No corepacks files were found, skipping uploading."
+  core_packs_files_count=$(find "${ARTIFACTS_FOLDER_SERVER_TYPE}" -name "corepacks*.json" | wc -l)
+  if [ "${core_packs_files_count}" -eq 0 ]; then
+    echo "No core packs files were found, skipping uploading."
   else
-    echo "Uploading corepacks files."
-    # Copy corepacks files from the artifacts folder to the build bucket:
-    find $ARTIFACTS_FOLDER -name "corepacks*.json" -exec gsutil cp -z json {} "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" \;
-    echo "Successfully uploaded corepacks files."
+    echo "Uploading ${core_packs_files_count} core packs files."
+    # Copy core packs files from the artifacts folder to the build bucket:
+    find "${ARTIFACTS_FOLDER_SERVER_TYPE}" -name "corepacks*.json" -exec gsutil cp -z json "{}" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH" \;
+    echo "Successfully uploaded core packs files."
   fi
 
-  if [ -f "$ARTIFACTS_FOLDER/versions-metadata.json" ]; then
+  if [ -f "${ARTIFACTS_FOLDER_SERVER_TYPE}/versions-metadata.json" ]; then
     echo "Uploading versions-metadata.json."
-    gsutil cp -z json "$ARTIFACTS_FOLDER/versions-metadata.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH"
+    gsutil cp -z json "${ARTIFACTS_FOLDER_SERVER_TYPE}/versions-metadata.json" "gs://$BUILD_BUCKET_PACKS_DIR_FULL_PATH"
     echo "Successfully uploaded versions-metadata.json."
   else
     echo "No versions-metadata.json file, skipping uploading."

diff --git a/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh b/Tests/scripts/uninstall_packs_and_reset_bucket_cloud.sh
@@ -25,13 +25,13 @@ fi
 if [[ -z "${CLOUD_CHOSEN_MACHINE_IDS}" ]]; then
   exit_on_error 1 "CLOUD_CHOSEN_MACHINE_IDS is not defined"
 else
-  gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER}/logs/gcloud_auth.log" 2>&1
+  gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER_INSTANCE}/logs/gcloud_auth.log" 2>&1
   exit_on_error $? "Failed to authenticate with GCS_MARKET_KEY"
 
   IFS=', ' read -r -a CLOUD_CHOSEN_MACHINE_ID_ARRAY <<< "${CLOUD_CHOSEN_MACHINE_IDS}"
   for CLOUD_CHOSEN_MACHINE_ID in "${CLOUD_CHOSEN_MACHINE_ID_ARRAY[@]}"; do
     echo "Copying prod bucket to ${CLOUD_CHOSEN_MACHINE_ID} bucket."
-    gsutil -m cp -r "gs://$GCS_SOURCE_BUCKET/content" "gs://$GCS_MACHINES_BUCKET/${CLOUD_CHOSEN_MACHINE_ID}/" > "$ARTIFACTS_FOLDER/Copy_prod_bucket_to_cloud_machine_cleanup.log" 2>&1
+    gsutil -m cp -r "gs://$GCS_SOURCE_BUCKET/content" "gs://$GCS_MACHINES_BUCKET/${CLOUD_CHOSEN_MACHINE_ID}/" >> "${ARTIFACTS_FOLDER_INSTANCE}/logs/Copy_prod_bucket_to_cloud_machine_cleanup.log" 2>&1
     exit_on_error $? "Failed to copy prod bucket to ${CLOUD_CHOSEN_MACHINE_ID} bucket"
   done