Create variant builds in batches (#3592)

Task/Issue URL: https://app.asana.com/0/1199230911884351/1208823340768320/f

Description:
This change adjusts the variants script to produce batches of up to 256 variants (currently 2, hardcoded)
and passes them to a (hardcoded) 2 concurrent batch tasks in the GHA workflow.

.github/actions/asana-get-build-variants-list/action.yml

@@ -19,9 +19,12 @@ inputs:
     required: true
     type: string
 outputs:
-  build-variants:
-    description: "The list of build variants to create"
-    value: ${{ steps.get-build-variants-task.outputs.build-variants }}
+  build-variants-1:
+    description: "The list of build variants to create (batch 1)"
+    value: ${{ steps.get-build-variants-task.outputs.build-variants-1 }}
+  build-variants-2:
+    description: "The list of build variants to create (batch 2)"
+    value: ${{ steps.get-build-variants-task.outputs.build-variants-2 }}
 runs:
   using: "composite"
   steps:

.github/actions/asana-get-build-variants-list/get_build_variants_list.sh

@@ -22,9 +22,7 @@ _create_origins_and_variants() {
 	jq -c '.data[]
 		| select(.custom_fields[] | select(.name == "'"${origin_field}"'").text_value != null)
 		| {origin: (.custom_fields[] | select(.name == "'"${origin_field}"'") | .text_value), variant: (.custom_fields[] | select(.name == "'"${atb_field}"'") | .text_value)}
-		| if .variant != null then {origin}, {origin, variant} else {origin} end' <<< "$response" \
-		| tr '\n' ',' \
-		| sed 's/,$//'
+		| if .variant != null then {origin}, {origin, variant} else {origin} end' <<< "$response"
 }
 
 # Fetch all the Asana tasks in the section specified by ORIGIN_ASANA_SECTION_ID for a project.
@@ -56,7 +54,7 @@ _fetch_origin_tasks() {
 		fi
 	done
 
-	echo "${origin_variants[*]}" | tr ' ' ','
+	printf "%s\n" "${origin_variants[@]}"
 }
 
 # Create a JSON string from the list of ATB items passed.
@@ -70,9 +68,7 @@ _create_atb_variant_pairs() {
 	# remove the trailing comma at the end of the line.
 	jq -R -c 'split(",") 
 		| map({variant: .}) 
-		| .[]' <<< "$response" \
-		| tr '\n' ',' \
-		| sed 's/,$//'
+		| .[]' <<< "$response"
 }
 
 # Fetches all the ATB variants defined in the ATB_ASANA_TASK_ID at the Variants list (comma separated) section.
@@ -92,21 +88,53 @@ _fetch_atb_variants() {
 
 	variants_list=("$(_create_atb_variant_pairs "$atb_variants")")
 
-	echo "${variants_list[*]}" | tr ' ' ','
+	printf "%s\n" "${variants_list[@]}"
+}
+
+split_array_into_chunks() {
+    local array=("$@")
+    local chunk_size=256
+    local total_elements=${#array[@]}
+    local chunks=()
+	local items
+
+    for ((i = 0; i < total_elements; i += chunk_size)); do
+		# Format the list of variants in a JSON object suitable for being consumed by GitHub Actions matrix.
+		# For more info see https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations.
+		items="$(echo "${array[@]:i:chunk_size}" |  tr ' ' ',')"
+		chunks+=("{\"include\": [${items}]}")
+    done
+
+	printf "%s\n" "${chunks[@]}"
 }
 
 main() {
-	local atb_variants
-	local origin_variants
+	local variants=()
+	local items=()
+
 	# fetch ATB variants
-	atb_variants="$(_fetch_atb_variants)"
+	variants+=("$(_fetch_atb_variants)")
 	# fetch Origin variants
-	origin_variants="$(_fetch_origin_tasks)"
-	# merges the two list together. Use `include` keyword for later usage in matrix. 
-	# for more info see https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations.
-	local merged_variants="{\"include\": [${atb_variants},${origin_variants}]}"
-	# write in GitHub output
-	echo "build-variants=${merged_variants}" >> "$GITHUB_OUTPUT"
+	variants+=("$(_fetch_origin_tasks "$origin_batch")")
+
+	while read -r variant; do
+		items+=("$variant")
+	done <<< "$(printf "%s\n" "${variants[@]}")"
+
+	echo "Found ${#items[@]} variants"
+
+	local chunks=()
+	while read -r chunk; do
+		chunks+=("$chunk")
+	done <<< "$(split_array_into_chunks "${items[@]}")"
+
+	local i=1
+    for chunk in "${chunks[@]}"; do
+		# Save to GitHub output
+		echo "Storing chunk #${i}"
+		echo "build-variants-${i}=${chunk}" >> "$GITHUB_OUTPUT"
+		i=$((i + 1))
+    done
 }
 
-main 
+main "$@"

.github/workflows/build_appstore.yml

@@ -77,7 +77,7 @@ jobs:
         esac
 
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
 

.github/workflows/build_notarized.yml

@@ -93,7 +93,7 @@ jobs:
 
     steps:
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
 

.github/workflows/create_variant.yml

@@ -1,3 +1,5 @@
+name: Create DMG Variant
+
 on:
   workflow_dispatch:
     inputs:
@@ -51,7 +53,7 @@ jobs:
     steps:
 
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
 

.github/workflows/create_variants.yml

@@ -36,7 +36,8 @@ jobs:
     timeout-minutes: 15
 
     outputs:
-      build-variants: ${{ steps.get-build-variants.outputs.build-variants }}
+      build-variants-1: ${{ steps.get-build-variants.outputs.build-variants-1 }}
+      build-variants-2: ${{ steps.get-build-variants.outputs.build-variants-2 }}
 
     steps:
       - name: Check out repository
@@ -69,14 +70,35 @@ jobs:
           path: ${{ github.workspace }}/duckduckgo.dmg
           retention-days: 1
 
-  create-variants:
+  create-variants-1:
 
     name: Create Variant
     needs: [set-up-variants, download-dmg-and-upload-artifact]
 
     strategy:
       fail-fast: false
-      matrix: ${{ fromJSON(needs.set-up-variants.outputs.build-variants) }}
+      matrix: ${{ fromJSON(needs.set-up-variants.outputs.build-variants-1) }}
+    uses: ./.github/workflows/create_variant.yml
+    with:
+      atb-variant: ${{ matrix.variant }}
+      origin-variant: ${{ matrix.origin }}
+    secrets:
+      APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
+      APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
+      APPLE_API_KEY_ISSUER: ${{ secrets.APPLE_API_KEY_ISSUER }}
+      AWS_ACCESS_KEY_ID_RELEASE_S3: ${{ secrets.AWS_ACCESS_KEY_ID_RELEASE_S3 }}
+      AWS_SECRET_ACCESS_KEY_RELEASE_S3: ${{ secrets.AWS_SECRET_ACCESS_KEY_RELEASE_S3 }}
+      MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
+      SSH_PRIVATE_KEY_FASTLANE_MATCH: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
+
+  create-variants-2:
+
+    name: Create Variant
+    needs: [set-up-variants, download-dmg-and-upload-artifact]
+
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.set-up-variants.outputs.build-variants-2) }}
     uses: ./.github/workflows/create_variant.yml
     with:
       atb-variant: ${{ matrix.variant }}
@@ -93,14 +115,29 @@ jobs:
   mattermost:
 
     name: Send Mattermost message
-    needs: create-variants
+    needs: [create-variants-1, create-variants-2]
     runs-on: macos-15
 
     env:
-      success: ${{ needs.create-variants.result == 'success' }}
-      failure: ${{ needs.create-variants.result == 'failure' }}
+      success: ${{ needs.create-variants-1.result == 'success' && needs.create-variants-2.result == 'success' }}
+      failure: ${{ needs.create-variants-1.result == 'failure' || needs.create-variants-2.result == 'failure' }}
 
     steps:
+      - name: Check out the code
+        if: ${{ env.success || env.failure }} # Don't execute when cancelled
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github
+            Gemfile
+            Gemfile.lock
+            fastlane
+            scripts
+
+      - name: Set up fastlane
+        if: ${{ env.success || env.failure }} # Don't execute when cancelled
+        run: bundle install
+
       - name: Send Mattermost message
         if: ${{ env.success || env.failure }} # Don't execute when cancelled
         env:

.github/workflows/pir_end_to_end_tests.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
     - name: Register SSH keys for certificates repository and PIR fake broker repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: |
                         ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}

.github/workflows/pr.yml

@@ -128,7 +128,7 @@ jobs:
 
     steps:
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
 
@@ -322,7 +322,7 @@ jobs:
 
     steps:
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
 

.github/workflows/sync_end_to_end.yml

@@ -56,7 +56,7 @@ jobs:
       run: screenresolution set 1920x1080x32@60
 
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
 

.github/workflows/sync_end_to_end_legacy_os.yml

@@ -42,7 +42,7 @@ jobs:
     timeout-minutes: 60
     steps:
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}
 

.github/workflows/ui_tests.yml

@@ -64,7 +64,7 @@ jobs:
       run: screenresolution set 1920x1080x32@60
 
     - name: Register SSH key for certificates repository access
-      uses: webfactory/ssh-agent@v0.7.0
+      uses: webfactory/ssh-agent@v0.9.0
       with:
         ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY_FASTLANE_MATCH }}