steel.yml

name: Steel

on:
  schedule:
    - cron: "0 0 * * *"
  push:
    branches:
      - main
  pull_request:
    types: [opened, synchronize, reopened]
    branches:
      - main

env:
  MAX_JOBS: 64
  MIN_PROJECTS_PER_JOB: 4
  MIN_PROJECTS_FOR_MATRIX: 4

jobs:
  changes:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      changed_projects: ${{ steps.analyze.outputs.changed_projects }}
      total_projects: ${{ steps.analyze.outputs.total_projects }}
      matrix: ${{ steps.matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: changes
        if: github.event_name == 'pull_request'
        with:
          list-files: shell
          filters: |
            steel:
              - added|modified: '**/steel/**'
            workflow:
              - added|modified: '.github/workflows/steel.yml'
      - name: Analyze Changes
        id: analyze
        run: |
          # Generate ignore pattern, excluding comments
          ignore_pattern=$(grep -v '^#' .github/.ghaignore | grep -v '^$' | tr '\n' '|' | sed 's/|$//')
          echo "Ignore pattern: $ignore_pattern"

          function get_projects() {
            find . -type d -name "steel" | grep -vE "$ignore_pattern" | sort
          }

          # Determine which projects to build and test
          if [[ "${{ github.event_name }}" == "push" || "${{ github.event_name }}" == "schedule" || "${{ steps.changes.outputs.workflow }}" == "true" ]]; then
            projects=$(get_projects)
          elif [[ "${{ steps.changes.outputs.steel }}" == "true" ]]; then
            changed_files=(${{ steps.changes.outputs.steel_files }})
            projects=$(for file in "${changed_files[@]}"; do dirname "${file}" | grep steel | sed 's#/steel/.*#/steel#g'; done | grep -vE "$ignore_pattern" | sort -u)
          else
            projects=""
          fi

          # Output project information
          if [[ -n "$projects" ]]; then
            echo "Projects to build and test"
            echo "$projects"
            total_projects=$(echo "$projects" | wc -l)
            echo "Total projects: $total_projects"
            echo "total_projects=$total_projects" >> $GITHUB_OUTPUT
            echo "changed_projects=$(echo "$projects" | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
          else
            echo "No projects to build and test."
            echo "total_projects=0" >> $GITHUB_OUTPUT
            echo "changed_projects=[]" >> $GITHUB_OUTPUT
          fi
      - name: Generate matrix
        id: matrix
        run: |
          total_projects=${{ steps.analyze.outputs.total_projects }}
          max_jobs=${{ env.MAX_JOBS }}
          min_projects_per_job=${{ env.MIN_PROJECTS_PER_JOB }}
          min_projects_for_matrix=${{ env.MIN_PROJECTS_FOR_MATRIX }}

          if [ "$total_projects" -lt "$min_projects_for_matrix" ]; then
            echo "matrix=[0]" >> $GITHUB_OUTPUT
          else
            projects_per_job=$(( (total_projects + max_jobs - 1) / max_jobs ))
            projects_per_job=$(( projects_per_job > min_projects_per_job ? projects_per_job : min_projects_per_job ))
            num_jobs=$(( (total_projects + projects_per_job - 1) / projects_per_job ))

            indices=$(seq 0 $(( num_jobs - 1 )))
            echo "matrix=[$(echo $indices | tr ' ' ',')]" >> $GITHUB_OUTPUT
          fi

  rust-checks:
    needs: changes
    if: ${{ github.event_name == 'pull_request' && needs.changes.outputs.total_projects != '0' }}
    name: Rust Checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy
      - name: Run sccache-cache
        if: github.event_name != 'release'
        uses: mozilla-actions/sccache-action@v0.0.6
      - name: Set Rust cache env vars
        if: github.event_name != 'release'
        run: |
          echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
      - name: Run fmt and clippy
        run: |
          readarray -t all_projects < <(echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]?')
          for project in "${all_projects[@]}"; do
            echo "::group::Checking ${project}"
            if [ ! -f "${project}/Cargo.toml" ]; then
              echo "::error::No Cargo.toml found in ${project}"
              exit 1
            fi
            cd "${project}"
            cargo fmt --check
            cargo clippy --all-features -- -D warnings
            cd - > /dev/null
            echo "::endgroup::"
          done

  build-and-test:
    needs: changes
    if: needs.changes.outputs.total_projects != '0'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        index: ${{ fromJson(needs.changes.outputs.matrix) }}
    name: build-and-test-group-${{ matrix.index }}
    outputs:
      failed_projects: ${{ steps.set-failed.outputs.failed_projects }}
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - name: Run sccache-cache
        if: github.event_name != 'release'
        uses: mozilla-actions/sccache-action@v0.0.6
      - name: Set Rust cache env vars
        if: github.event_name != 'release'
        run: |
          echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
          echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
      - uses: actions/cache@v3
        with:
          path: ~/.cargo/bin/steel
          key: ${{ runner.os }}-steel-cli
      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20.x
          check-latest: true
      - name: Setup build environment
        id: setup
        run: |
          npm install --global pnpm

          # Create the build and test function
          cat << 'EOF' > build_and_test.sh
          function build_and_test() {
            local project=$1
            local solana_version=$2
            echo "Building and Testing $project with Solana $solana_version"
            cd "$project" || return 1

            # Install dependencies
            if [ -f "package.json" ]; then
              if ! pnpm install --frozen-lockfile; then
                echo "::error::pnpm install failed for $project"
                echo "$project: pnpm install failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
                cd - > /dev/null
                return 1
              fi

              # Build
              if ! pnpm build; then
                echo "::error::build failed for $project"
                echo "$project: build failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
                cd - > /dev/null
                return 1
              fi

              # Test
              if ! pnpm build-and-test; then
                echo "::error::tests failed for $project"
                echo "$project: tests failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
                cd - > /dev/null
                return 1
              fi
            else
              # Use Steel CLI
              if ! cargo install --quiet steel-cli; then
                echo "::error::steel-cli installation failed for $project"
                echo "$project: steel-cli installation failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
                cd - > /dev/null
                return 1
              fi

              # Build
              if ! steel build; then
                echo "::error::steel build failed for $project"
                echo "$project: steel build failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
                cd - > /dev/null
                return 1
              fi

              # Test
              if ! steel test; then
                echo "::error::steel test failed for $project"
                echo "$project: steel test failed with $solana_version" >> $GITHUB_WORKSPACE/failed_projects.txt
                cd - > /dev/null
                return 1
              fi
            fi

            echo "Build and tests succeeded for $project with $solana_version version."
            cd - > /dev/null
            return 0
          }

          function process_projects() {
            local solana_version=$1

            readarray -t all_projects < <(echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]?')
            start_index=$(( ${{ matrix.index }} * ${{ env.MIN_PROJECTS_PER_JOB }} ))
            end_index=$(( start_index + ${{ env.MIN_PROJECTS_PER_JOB }} ))
            end_index=$(( end_index > ${{ needs.changes.outputs.total_projects }} ? ${{ needs.changes.outputs.total_projects }} : end_index ))

            echo "Projects to build and test in this job"
            for i in $(seq $start_index $(( end_index - 1 ))); do
              echo "${all_projects[$i]}"
            done

            failed=false
            for i in $(seq $start_index $(( end_index - 1 ))); do
              echo "::group::Building and testing ${all_projects[$i]}"
              if ! build_and_test "${all_projects[$i]}" "$solana_version"; then
                failed=true
              fi
              echo "::endgroup::"
            done

            return $([ "$failed" = true ] && echo 1 || echo 0)
          }
          EOF

          # Make the script executable
          chmod +x build_and_test.sh

      - name: Setup Solana stable
        uses: heyAyushh/setup-solana@v5.4
        with:
          solana-cli-version: stable
      - name: Build and Test with Stable
        env:
          SCCACHE_GHA_ENABLED: "true"
          RUSTC_WRAPPER: "sccache"
        run: |
          source build_and_test.sh
          solana -V
          rustc -V
          process_projects "stable"
          sccache --show-stats
      - name: Setup Solana 1.18.17
        uses: heyAyushh/setup-solana@v5.4
        with:
          solana-cli-version: 1.18.17
      - name: Build and Test with 1.18.17
        env:
          SCCACHE_GHA_ENABLED: "true"
          RUSTC_WRAPPER: "sccache"
        run: |
          source build_and_test.sh
          solana -V
          rustc -V
          process_projects "1.18.17"
          sccache --show-stats

      - name: Set failed projects output
        id: set-failed
        if: failure()
        run: |
          if [ -f "$GITHUB_WORKSPACE/failed_projects.txt" ]; then
            failed_projects=$(cat $GITHUB_WORKSPACE/failed_projects.txt | jq -R -s -c 'split("\n")[:-1]')
            echo "failed_projects=$failed_projects" >> $GITHUB_OUTPUT
          else
            echo "failed_projects=[]" >> $GITHUB_OUTPUT
          fi

  summary:
    needs: [changes, build-and-test]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Create job summary
        run: |
          echo "## Steel Workflow Summary" >> $GITHUB_STEP_SUMMARY
          echo "- Total projects: ${{ needs.changes.outputs.total_projects }}" >> $GITHUB_STEP_SUMMARY

          # List all processed projects
          echo "<details>" >> $GITHUB_STEP_SUMMARY
          echo "<summary>Projects processed (click to expand)</summary>" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo '${{ needs.changes.outputs.changed_projects }}' | jq -r '.[]' | while read project; do
            echo "- $project" >> $GITHUB_STEP_SUMMARY
          done
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "</details>" >> $GITHUB_STEP_SUMMARY

          # Report build and test results
          if [[ "${{ needs.build-and-test.result }}" == "failure" ]]; then
            echo "## :x: Build or tests failed" >> $GITHUB_STEP_SUMMARY
            echo "<details>" >> $GITHUB_STEP_SUMMARY
            echo "<summary>Failed projects (click to expand)</summary>" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            failed_projects='${{ needs.build-and-test.outputs.failed_projects }}'
            if [[ -n "$failed_projects" ]]; then
              echo "$failed_projects" | jq -r '.[]' | while IFS=: read -r project failure_reason; do
                echo "- **$project**" >> $GITHUB_STEP_SUMMARY
                echo "  - Failure reason: $failure_reason" >> $GITHUB_STEP_SUMMARY
              done
            else
              echo "No failed projects reported. This might indicate an unexpected error in the workflow." >> $GITHUB_STEP_SUMMARY
            fi
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "</details>" >> $GITHUB_STEP_SUMMARY
          elif [[ "${{ needs.build-and-test.result }}" == "success" ]]; then
            echo "## :white_check_mark: All builds and tests passed" >> $GITHUB_STEP_SUMMARY
          else
            echo "## :warning: Build and test job was skipped or canceled" >> $GITHUB_STEP_SUMMARY
          fi