Merge branch 'solana-developers:main' into steel/program-derived-addr…

…esses

.github/workflows/solana-native.yml

@@ -11,107 +11,246 @@ on:
     branches:
       - main
 
+env:
+  MAX_JOBS: 64
+  MIN_PROJECTS_PER_JOB: 4
+  MIN_PROJECTS_FOR_MATRIX: 4
+
 jobs:
-  build:
+  changes:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [20.x]
-        solana-version: [stable]
+    permissions:
+      pull-requests: read
+    outputs:
+      changed_projects: ${{ steps.analyze.outputs.changed_projects }}
+      total_projects: ${{ steps.analyze.outputs.total_projects }}
+      matrix: ${{ steps.matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
+      - uses: dorny/paths-filter@v3
+        id: changes
+        if: github.event_name == 'pull_request'
         with:
-          node-version: ${{ matrix.node-version }}
-          check-latest: true
-      - uses: heyAyushh/[email protected]
-        with:
-          solana-cli-version: ${{ matrix.solana-version }}
-      - run: solana -V
-        shell: bash
-      - name: Install pnpm
+          list-files: shell
+          filters: |
+            native:
+              - added|modified: '**/native/**'
+            workflow:
+              - added|modified: '.github/workflows/solana-native.yml'
+      - name: Analyze Changes
+        id: analyze
         run: |
-          npm install --global pnpm
-      - name: Build Native programs
+          # Generate ignore pattern, excluding comments
+          ignore_pattern=$(grep -v '^#' .github/.ghaignore | grep -v '^$' | tr '\n' '|' | sed 's/|$//')
+          echo "Ignore pattern: $ignore_pattern"
+
+          function get_projects() {
+            find . -type d -name "native" | grep -vE "$ignore_pattern" | sort
+          }
+
+          # Determine which projects to build and test
+          if [[ "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "schedule" || "${{ steps.changes.outputs.workflow }}" == "true" ]]; then
+            projects=$(get_projects)
+          elif [[ "${{ steps.changes.outputs.native }}" == "true" ]]; then
+            changed_files=(${{ steps.changes.outputs.native_files }})
+            projects=$(for file in "${changed_files[@]}"; do dirname "${file}" | grep native | sed 's#/native/.*#/native#g'; done | grep -vE "$ignore_pattern" | sort -u)
+          else
+            projects=""
+          fi
+
+          # Output project information
+          if [[ -n "$projects" ]]; then
+            echo "Projects to build and test"
+            echo "$projects"
+            total_projects=$(echo "$projects" | wc -l)
+            echo "Total projects: $total_projects"
+            echo "total_projects=$total_projects" >> $GITHUB_OUTPUT
+            echo "changed_projects=$(echo "$projects" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+          else
+            echo "No projects to build and test."
+            echo "total_projects=0" >> $GITHUB_OUTPUT
+            echo "changed_projects=[]" >> $GITHUB_OUTPUT
+          fi
+      - name: Generate matrix
+        id: matrix
         run: |
-          declare -a ProjectDirs=($(find . -type d -name "native"| grep -v -f <(grep . .github/.ghaignore | grep -v '^$')))
-          echo "Projects to Build:"
-          printf "%s\n" "${ProjectDirs[@]}"
-          for projectDir in "${ProjectDirs[@]}"; do
-            echo "
-            ********
-            Building $projectDir
-            ********"
-            cd $projectDir
-            if pnpm build; then
-              echo "Build succeeded for $projectDir."
-            else
-              failed=true
-              failed_builds+=($projectDir)
-              echo "Build failed for $projectDir. Continuing with the next program."
-            fi
-          cd - > /dev/null
-          done
-          if [ "$failed" = true ]; then
-            echo "Programs that failed building:"
-            printf "%s\n" "${failed_builds[@]}"
-            exit 1
+          total_projects=${{ steps.analyze.outputs.total_projects }}
+          max_jobs=${{ env.MAX_JOBS }}
+          min_projects_per_job=${{ env.MIN_PROJECTS_PER_JOB }}
+          min_projects_for_matrix=${{ env.MIN_PROJECTS_FOR_MATRIX }}
+
+          if [ "$total_projects" -lt "$min_projects_for_matrix" ]; then
+            echo "matrix=[0]" >> $GITHUB_OUTPUT
           else
-            echo "All programs built successfully."
+            projects_per_job=$(( (total_projects + max_jobs - 1) / max_jobs ))
+            projects_per_job=$(( projects_per_job > min_projects_per_job ? projects_per_job : min_projects_per_job ))
+            num_jobs=$(( (total_projects + projects_per_job - 1) / projects_per_job ))
+
+            indices=$(seq 0 $(( num_jobs - 1 )))
+            echo "matrix=[$(echo $indices | tr ' ' ',')]" >> $GITHUB_OUTPUT
           fi
-        shell: bash
 
-  test:
+  build-and-test:
+    needs: changes
+    if: needs.changes.outputs.total_projects != '0'
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        node-version: [20.x]
-        solana-version: [1.18.17, stable]
+        index: ${{ fromJson(needs.changes.outputs.matrix) }}
+    name: build-and-test-group-${{ matrix.index }}
+    outputs:
+      failed_projects: ${{ steps.set-failed.outputs.failed_projects }}
     steps:
       - uses: actions/checkout@v4
-      - name: Use Node.js ${{ matrix.node-version }}
+      - name: Use Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: ${{ matrix.node-version }}
+          node-version: 20.x
           check-latest: true
-      - uses: heyAyushh/[email protected]
-        with:
-          solana-cli-version: ${{ matrix.solana-version }}
-      - run: solana block
-        shell: bash
-      - name: Install pnpm
+      - name: Setup build environment
+        id: setup
         run: |
+          # Create the build and test function
+          cat << 'EOF' > build_and_test.sh
+          function build_and_test() {
+            local project=$1
+            local solana_version=$2
+            echo "Building and Testing $project with Solana $solana_version"
+            cd "$project" || return 1
+
+            # Install dependencies
+            if ! pnpm install --frozen-lockfile; then
+              echo "::error::pnpm install failed for $project"
+              echo "$project: pnpm install failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
+              cd - > /dev/null
+              return 1
+            fi
+
+            # Build
+            if ! pnpm build; then
+              echo "::error::build failed for $project"
+              echo "$project: build failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
+              cd - > /dev/null
+              return 1
+            fi
+
+            # Test
+            if ! pnpm build-and-test; then
+              echo "::error::tests failed for $project"
+              echo "$project: tests failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
+              cd - > /dev/null
+              return 1
+            fi
+
+            echo "Build and tests succeeded for $project with $solana_version version."
+            cd - > /dev/null
+            return 0
+          }
+
+          function process_projects() {
+            local solana_version=$1
+
+            readarray -t all_projects < <(echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]?')
+            start_index=$(( ${{ matrix.index }} * ${{ env.MIN_PROJECTS_PER_JOB }} ))
+            end_index=$(( start_index + ${{ env.MIN_PROJECTS_PER_JOB }} ))
+            end_index=$(( end_index > ${{ needs.changes.outputs.total_projects }} ? ${{ needs.changes.outputs.total_projects }} : end_index ))
+
+            echo "Projects to build and test in this job"
+            for i in $(seq $start_index $(( end_index - 1 ))); do
+              echo "${all_projects[$i]}"
+            done
+
+            failed=false
+            for i in $(seq $start_index $(( end_index - 1 ))); do
+              echo "::group::Building and testing ${all_projects[$i]}"
+              if ! build_and_test "${all_projects[$i]}" "$solana_version"; then
+                failed=true
+              fi
+              echo "::endgroup::"
+            done
+
+            return $([ "$failed" = true ] && echo 1 || echo 0)
+          }
+          EOF
+
+          # Make the script executable
+          chmod +x build_and_test.sh
+
+          # Install pnpm
           npm install --global pnpm
-      - name: Test solana native programs
+      - name: Setup Solana stable
+        uses: heyAyushh/[email protected]
+        with:
+          solana-cli-version: stable
+      - name: Build and Test with Stable
+        run: |
+          source build_and_test.sh
+          solana -V
+          rustc -V
+          process_projects "stable"
+      - name: Setup Solana 1.18.17
+        uses: heyAyushh/[email protected]
+        with:
+          solana-cli-version: 1.18.17
+      - name: Build and Test with 1.18.17
         run: |
+          source build_and_test.sh
           solana -V
           rustc -V
-          declare -a ProjectDirs=($(find . -type d -name "native"| grep -v -f <(grep . .github/.ghaignore | grep -v '^$')))
-          echo "Projects to Test:"
-          printf "%s\n" "${ProjectDirs[@]}"
-          for projectDir in "${ProjectDirs[@]}"; do
-            echo "
-            ********
-            Testing $projectDir
-            ********"
-            cd $projectDir
-            pnpm install --frozen-lockfile
-            if pnpm build-and-test; then
-              echo "Tests succeeded for $projectDir."
+          process_projects "1.18.17"
+
+      - name: Set failed projects output
+        id: set-failed
+        if: failure()
+        run: |
+          if [ -f "$GITHUB_WORKSPACE/failed_projects.txt" ]; then
+            failed_projects=$(cat $GITHUB_WORKSPACE/failed_projects.txt | jq -R -s -c 'split("\n")[:-1]')
+            echo "failed_projects=$failed_projects" >> $GITHUB_OUTPUT
+          else
+            echo "failed_projects=[]" >> $GITHUB_OUTPUT
+          fi
+
+  summary:
+    needs: [changes, build-and-test]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create job summary
+        run: |
+          echo "## Native Workflow Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- Total projects: ${{ needs.changes.outputs.total_projects }}" >> $GITHUB_STEP_SUMMARY
+
+          # List all processed projects
+          echo "<details>" >> $GITHUB_STEP_SUMMARY
+          echo "<summary>Projects processed (click to expand)</summary>" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]' | while read project; do
+            echo "- $project" >> $GITHUB_STEP_SUMMARY
+          done
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
+
+          # Report build and test results
+          if [[ "${{ needs.build-and-test.result }}" == "failure" ]]; then
+            echo "## :x: Build or tests failed" >> $GITHUB_STEP_SUMMARY
+            echo "<details>" >> $GITHUB_STEP_SUMMARY
+            echo "<summary>Failed projects (click to expand)</summary>" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            failed_projects='${{ needs.build-and-test.outputs.failed_projects }}'
+            if [[ -n "$failed_projects" ]]; then
+              echo "$failed_projects" | jq -r '.[]' | while IFS=: read -r project failure_reason; do
+                echo "- **$project**" >> $GITHUB_STEP_SUMMARY
+                echo "  - Failure reason: $failure_reason" >> $GITHUB_STEP_SUMMARY
+              done
             else
-              failed=true
-              failed_tests+=($projectDir)
-              echo "Tests failed for $projectDir. Continuing with the next program."
+              echo "No failed projects reported. This might indicate an unexpected error in the workflow." >> $GITHUB_STEP_SUMMARY
             fi
-          cd - > /dev/null
-          done
-          if [ "$failed" = true ]; then
-            echo "*****************************"
-            echo "Programs that failed testing:"
-            printf "%s\n" "${failed_tests[@]}"
-            exit 1
+            echo "" >> $GITHUB_STEP_SUMMARY
+            echo "</details>" >> $GITHUB_STEP_SUMMARY
+          elif [[ "${{ needs.build-and-test.result }}" == "success" ]]; then
+            echo "## :white_check_mark: All builds and tests passed" >> $GITHUB_STEP_SUMMARY
           else
-            echo "All tests passed."
+            echo "## :warning: Build and test job was skipped or canceled" >> $GITHUB_STEP_SUMMARY
           fi
-        shell: bash