e2e-matrix.yml

name: E2E matrix

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      debug_enabled:
        type: boolean
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false

env:
  CYPRESS_CACHE_FOLDER: ${{ github.workspace }}/.cypress

permissions: {}
jobs:
  preinstall:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
        node_version:
          - 19
          - 18
          - 16
        exclude:
          # run just node v18 on macos
          - os: macos-latest
            node_version: 19
          - os: macos-latest
            node_version: 16

    name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }})
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install PNPM
        run: |
          npm install -g @pnpm/exe@8.3.1

      - name: Set node
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node_version }}
          cache: 'pnpm'

      - name: Cache node_modules
        id: cache-modules
        uses: actions/cache@v3
        with:
          lookup-only: true
          path: '**/node_modules'
          key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ github.run_id }}

      - name: Install packages
        if: steps.cache-modules.outputs.cache-hit != 'true'
        run: pnpm install --frozen-lockfile

      - name: Homebrew cache directory path
        if: ${{ matrix.os == 'macos-latest' }}
        id: homebrew-cache-dir-path
        run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT

      - name: Cache Homebrew
        if: ${{ matrix.os == 'macos-latest' }}
        uses: actions/cache@v3
        with:
          lookup-only: true
          path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
          key: brew-${{ matrix.node_version }}
          restore-keys: |
            brew-

      - name: Cache Cypress
        id: cache-cypress
        uses: actions/cache@v3
        with:
          lookup-only: true
          path: '${{ github.workspace }}/.cypress'
          key: ${{ runner.os }}-cypress

      - name: Install Cypress
        if: steps.cache-cypress.outputs.cache-hit != 'true'
        run: npx cypress install

  e2e:
    needs: preinstall
    permissions:
      contents: read
    runs-on: ${{ matrix.os }}
    timeout-minutes: 90
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
        node_version:
          - 19
          - 18
          - 16
        package_manager:
          - npm
          - yarn
          - pnpm
        project:
          - e2e-angular-core
          - e2e-angular-extensions
          - e2e-cypress
          - e2e-detox
          - e2e-esbuild
          - e2e-expo
          - e2e-jest
          - e2e-js
          - e2e-lerna-smoke-tests
          - e2e-linter
          - e2e-next
          - e2e-node
          - e2e-nx-init
          - e2e-nx-misc
          - e2e-nx-plugin
          - e2e-nx-run
          - e2e-react-core
          - e2e-react-extensions
          - e2e-react-native
          - e2e-web
          - e2e-rollup
          - e2e-storybook
          - e2e-storybook-angular
          - e2e-vite
          - e2e-webpack
          - e2e-workspace-create
          - e2e-workspace-create-npm
        include:
          # os short names
          - os: ubuntu-latest
            os_name: 'Linux'
          - os: macos-latest
            os_name: 'MacOS'
          # test timeouts
          - os: ubuntu-latest
            os_timeout: 60
          - os: macos-latest
            os_timeout: 90
          # codeowner groups
          - project: e2e-angular-core
            codeowners: 'S04SS457V38'
          - project: e2e-angular-extensions
            codeowners: 'S04SS457V38'
          - project: e2e-cypress
            codeowners: 'S04T16BTJJY'
          - project: e2e-detox
            codeowners: 'S04TNCNJG5N'
          - project: e2e-esbuild
            codeowners: 'S04SJ6HHP0X'
          - project: e2e-expo
            codeowners: 'S04TNCNJG5N'
          - project: e2e-jest
            codeowners: 'S04T16BTJJY'
          - project: e2e-js
            codeowners: 'S04SJ6HHP0X'
          - project: e2e-lerna-smoke-tests
            codeowners: 'S04TNCVEETS'
          - project: e2e-linter
            codeowners: 'S04SYJGKSCT'
          - project: e2e-next
            codeowners: 'S04TNCNJG5N'
          - project: e2e-node
            codeowners: 'S04SJ6HHP0X'
          - project: e2e-nx-init
            codeowners: 'S04SYHYKGNP'
          - project: e2e-nx-misc
            codeowners: 'S04SYHYKGNP'
          - project: e2e-nx-plugin
            codeowners: 'S04SYHYKGNP'
          - project: e2e-nx-run
            codeowners: 'S04SYHYKGNP'
          - project: e2e-react-core
            codeowners: 'S04TNCNJG5N'
          - project: e2e-react-extensions
            codeowners: 'S04TNCNJG5N'
          - project: e2e-react-native
            codeowners: 'S04TNCNJG5N'
          - project: e2e-web
            codeowners: 'S04SJ6PL98X'
          - project: e2e-rollup
            codeowners: 'S04SJ6PL98X'
          - project: e2e-storybook
            codeowners: 'S04SVQ8H0G5'
          - project: e2e-storybook-angular
            codeowners: 'S04SVQ8H0G5'
          - project: e2e-vite
            codeowners: 'S04SJ6PL98X'
          - project: e2e-webpack
            codeowners: 'S04SJ6PL98X'
          - project: e2e-workspace-create
            codeowners: 'S04SYHYKGNP'
          - project: e2e-workspace-create-npm
            codeowners: 'S04SYHYKGNP'
        exclude:
          # exclude react-native tests from ubuntu
          - os: ubuntu-latest
            project: e2e-react-native
          - os: ubuntu-latest
            project: e2e-detox
          - os: ubuntu-latest
            project: e2e-expo
          # exclude non-CNW/Lerna tests from non-LTS node versions
          - node_version: 16
            project: e2e-angular-core
          - node_version: 16
            project: e2e-angular-extensions
          - node_version: 16
            project: e2e-cypress
          - node_version: 16
            project: e2e-detox
          - node_version: 16
            project: e2e-esbuild
          - node_version: 16
            project: e2e-expo
          - node_version: 16
            project: e2e-jest
          - node_version: 16
            project: e2e-js
          - node_version: 16
            project: e2e-linter
          - node_version: 16
            project: e2e-next
          - node_version: 16
            project: e2e-node
          - node_version: 16
            project: e2e-nx-init
          - node_version: 16
            project: e2e-nx-misc
          - node_version: 16
            project: e2e-nx-plugin
          - node_version: 16
            project: e2e-lerna-smoke-tests
          - node_version: 16
            project: e2e-react-core
          - node_version: 16
            project: e2e-react-extensions
          - node_version: 16
            project: e2e-react-native
          - node_version: 16
            project: e2e-web
          - node_version: 16
            project: e2e-rollup
          - node_version: 16
            project: e2e-storybook
          - node_version: 16
            project: e2e-storybook-angular
          - node_version: 16
            project: e2e-vite
          - node_version: 16
            project: e2e-webpack
          - node_version: 19
            project: e2e-angular-core
          - node_version: 19
            project: e2e-angular-extensions
          - node_version: 19
            project: e2e-cypress
          - node_version: 19
            project: e2e-detox
          - node_version: 19
            project: e2e-esbuild
          - node_version: 19
            project: e2e-expo
          - node_version: 19
            project: e2e-jest
          - node_version: 19
            project: e2e-js
          - node_version: 19
            project: e2e-linter
          - node_version: 19
            project: e2e-next
          - node_version: 19
            project: e2e-node
          - node_version: 19
            project: e2e-nx-init
          - node_version: 19
            project: e2e-nx-misc
          - node_version: 19
            project: e2e-nx-plugin
          - node_version: 19
            project: e2e-lerna-smoke-tests
          - node_version: 19
            project: e2e-react-core
          - node_version: 19
            project: e2e-react-extensions
          - node_version: 19
            project: e2e-react-native
          - node_version: 19
            project: e2e-web
          - node_version: 19
            project: e2e-rollup
          - node_version: 19
            project: e2e-storybook
          - node_version: 19
            project: e2e-storybook-angular
          - node_version: 19
            project: e2e-vite
          - node_version: 19
            project: e2e-webpack
          # run just npm v18 on macos
          - os: macos-latest
            package_manager: yarn
          - os: macos-latest
            package_manager: pnpm
          - os: macos-latest
            node_version: 16
          - os: macos-latest
            node_version: 19
      fail-fast: false

    name: ${{ matrix.os_name }}/${{ matrix.package_manager }}/${{ matrix.node_version }} ${{ join(matrix.project) }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Prepare dir for output
        run: mkdir -p outputs

      - name: Install PNPM
        run: |
          npm install -g @pnpm/exe@8.3.1

      - name: Use Node.js ${{ matrix.node_version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node_version }}
          cache: 'pnpm'

      - name: Cache node_modules
        id: cache-modules
        uses: actions/cache@v3
        with:
          path: '**/node_modules'
          key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ github.run_id }}

      - name: Install packages
        run: pnpm install --frozen-lockfile

      - name: Cleanup
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          # Workaround to provide additional free space for testing.
          # https://github.com/actions/virtual-environments/issues/2840
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          sudo apt-get install lsof
          echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p

      - name: Homebrew cache directory path
        if: ${{ matrix.os == 'macos-latest' }}
        id: homebrew-cache-dir-path
        run: echo "dir=$(brew --cache)" >> $GITHUB_OUTPUT

      - name: Cache Homebrew
        if: ${{ matrix.os == 'macos-latest' }}
        uses: actions/cache@v3
        with:
          path: ${{ steps.homebrew-cache-dir-path.outputs.dir }}
          key: brew-${{ matrix.node_version }}
          restore-keys: |
            brew-

      - name: Cache Cypress
        id: cache-cypress
        uses: actions/cache@v3
        with:
          path: '${{ github.workspace }}/.cypress'
          key: ${{ runner.os }}-cypress

      - name: Install Cypress
        if: steps.cache-cypress.outputs.cache-hit != 'true'
        run: npx cypress install

      - name: Install applesimutils, reset ios simulators
        if: ${{ matrix.os == 'macos-latest' }}
        run: |
          HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null
          HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null
          xcrun simctl shutdown all && xcrun simctl erase all

      - name: Configure git metadata (needed for lerna smoke tests)
        run: |
          git config --global user.email test@test.com
          git config --global user.name "Test Test"

      - name: Set starting timestamp
        id: before-e2e
        run: |
          echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Run e2e tests
        id: e2e-run
        run: pnpm nx run-many -t e2e,e2e-macos -p ${{ matrix.project }}
        timeout-minutes: ${{ matrix.os_timeout }}
        env:
          GIT_AUTHOR_EMAIL: test@test.com
          GIT_AUTHOR_NAME: Test
          GIT_COMMITTER_EMAIL: test@test.com
          GIT_COMMITTER_NAME: Test
          NX_E2E_CI_CACHE_KEY: e2e-gha-${{ matrix.os }}-${{ matrix.node_version }}-${{ matrix.package_manager }}
          NODE_OPTIONS: --max_old_space_size=8192
          SELECTED_PM: ${{ matrix.package_manager }}
          npm_config_registry: http://localhost:4872
          YARN_REGISTRY: http://localhost:4872
          NX_CACHE_DIRECTORY: 'tmp'
          NX_E2E_SKIP_BUILD_CLEANUP: 'true'
          NX_E2E_RUN_E2E: 'true'
          NX_E2E_VERBOSE_LOGGING: 'true'
          NX_PERF_LOGGING: 'false'
          NX_DAEMON: 'true'

      - name: Save matrix config in file
        if: ${{ always() }}
        id: save-matrix
        shell: bash
        run: |
          before=${{ steps.before-e2e.outputs.timestamp }}
          now=$(date +%s)
          delta=$(($now - $before))
          matrix=$((
            echo '${{ toJSON(matrix) }}'
          ) | jq --argjson delta $delta -c '. + { "status": "${{ steps.e2e-run.outcome}}", "duration": $delta }')
          echo "$matrix" > matrix
          path=outputs/${{ matrix.os_name}}-${{ matrix.node_version}}-${{ matrix.package_manager}}-${{ matrix.project }}
          echo "path=$path" >> $GITHUB_OUTPUT
          echo "$matrix" > $path

      - name: Upload matrix config
        uses: actions/upload-artifact@v3
        if: ${{ always() }}
        with:
          name: outputs
          path: ${{ steps.save-matrix.outputs.path }}

      - name: Setup tmate session
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled && failure() }}
        uses: mxschmitt/action-tmate@v3.8
        timeout-minutes: 15

  process-result:
    if: ${{ always() }}
    runs-on: ubuntu-latest
    needs: e2e
    outputs:
      message: ${{ steps.process-json.outputs.SLACK_MESSAGE }}
      proj-duration: ${{ steps.process-json.outputs.SLACK_PROJ_DURATION }}
      pm-duration: ${{ steps.process-json.outputs.SLACK_PM_DURATION }}
      codeowners: ${{ steps.process-json.outputs.CODEOWNERS }}
    steps:
      - name: Load outputs
        uses: actions/download-artifact@v3
        with:
          name: outputs
          path: outputs

      - name: Join and stringify matrix configs
        id: combine-json
        run: |
          combined=$((jq -s . outputs/*) | jq tostring)
          echo "combined=$combined" >> $GITHUB_OUTPUT

      - name: Make slack outputs
        id: process-json
        uses: actions/github-script@v6
        env:
          GITHUB_TOKEN: ${{ github.token }}
        with:
          script: |
            const combined = JSON.parse(${{ steps.combine-json.outputs.combined }});
            const failedProjects = combined.filter(c => c.status === 'failure').sort((a, b) => a.project.localeCompare(b.project));

            // codeowners
            const codeowners = new Set();
            failedProjects.forEach(c => {
              codeowners.add(c.codeowners);
            });
            core.setOutput('CODEOWNERS', Array.from(codeowners).join(','));

            function trimSpace(res) {
              return res.split('\n').map((l) => l.trim()).join('\n');
            }

            // failed message
            let lastProject;
            let result = `
              \`\`\`
              | Failed project                 | PM   | OS    | Node |
              |--------------------------------|------|-------|------|`;
            failedProjects.forEach(matrix => {
              const project = matrix.project !== lastProject ? matrix.project : '...';
              result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name} | v${matrix.node_version.toString().padEnd(3)} |`
              lastProject = matrix.project;
            });
            result += `\`\`\``;
            core.setOutput('SLACK_MESSAGE', trimSpace(result));

            function humanizeDuration(num) {
              let res = '';
              const hours = Math.floor(num / 3600);
              if (hours) {
                res += `${hours}h `;
              }
              const mins = Math.floor((num % 3600) / 60);
              if (mins) {
                res += `${mins}m `;
              }
              const sec = num % 60;
              if (sec) {
                res += `${sec}s`
              }
              return res;
            }

            // duration message
            const timeReport = {};
            const pmReport = {
              npm: 0,
              yarn: 0,
              pnpm: 0
            };
            const macosProjects = ['e2e-detox', 'e2e-expo', 'e2e-react-native'];
            combined.forEach((matrix) => {
              if (matrix.os_name === 'Linux' && matrix.node_version === 18) {
                pmReport[matrix.package_manager] += matrix.duration;
              }
              if (matrix.os_name === 'Linux' || macosProjects.includes(matrix.project)) {
                if (timeReport[matrix.project]) {
                  if (matrix.duration > timeReport[matrix.project].max) {
                    timeReport[matrix.project].max = matrix.duration;
                    timeReport[
                      matrix.project
                    ].maxEnv = `${matrix.os_name}, ${matrix.package_manager}`;
                  }
                  if (matrix.duration < timeReport[matrix.project].min) {
                    timeReport[matrix.project].min = matrix.duration;
                    timeReport[
                      matrix.project
                    ].minEnv = `${matrix.os_name}, ${matrix.package_manager}`;
                  }
                } else {
                  timeReport[matrix.project] = {
                    min: matrix.duration,
                    max: matrix.duration,
                    minEnv: `${matrix.os_name}, ${matrix.package_manager}`,
                    maxEnv: `${matrix.os_name}, ${matrix.package_manager}`,
                  };
                }
              }
            });

            // project time report
            let resultPkg = `
                \`\`\`
                | Project                        | Time                      |
                |--------------------------------|---------------------------|`;
            function mapProjectTime(proj, section) {
              let res = '';
              res += `${humanizeDuration(timeReport[proj][section])}`;
              res += ` (${timeReport[proj][section + 'Env']})`
              return res;
            }
            function durationIcon(proj, section) {
              if (timeReport[proj][section] < 12 * 60) {
                return `${section} ✅`;
              }
              if (timeReport[proj][section] < 15 * 60) {
                return `${section} ❗`;
              }
              return `${section} ❌`;
            }
            Object.keys(timeReport).forEach(proj => {
              resultPkg += `\n| ${proj.padEnd(30)} |                           |`;
              resultPkg += `\n| ${durationIcon(proj, 'min').padStart(29)} | ${mapProjectTime(proj, 'min').padEnd(25)} |`;
              resultPkg += `\n| ${durationIcon(proj, 'max').padStart(29)} | ${mapProjectTime(proj, 'max').padEnd(25)} |`;
            });
            resultPkg += `\`\`\``;
            core.setOutput('SLACK_PROJ_DURATION', trimSpace(resultPkg));

            // Print project duration report inline to allow reviewing on manual runs (when no slack message will be sent)
            console.log(trimSpace(resultPkg));

            let resultPm = `
                \`\`\`
                | PM   | Total time  |
                |------|-------------|`;
            Object.keys(pmReport).forEach(pm => {
              resultPm += `\n| ${pm.padEnd(4)} | ${humanizeDuration(pmReport[pm]).padEnd(11)} |`
            });
            resultPm += `\`\`\``;
            core.setOutput('SLACK_PM_DURATION', trimSpace(resultPm));

            // Print package manager duration report inline to allow reviewing on manual runs (when no slack message will be sent)
            console.log(trimSpace(resultPm));

  report-failure:
    if: ${{ failure() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    name: Report failure
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: 'failure'
          message_format: '{emoji} Workflow has {status_message} ${{ needs.process-result.outputs.message }}'
          notification_title: '{workflow}'
          footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
          mention_groups: ${{ needs.process-result.outputs.codeowners }}
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-success:
    if: ${{ success() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: e2e
    runs-on: ubuntu-latest
    name: Report status
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: ${{ needs.e2e.result }}
          message_format: '{emoji} Workflow has {status_message}'
          notification_title: '{workflow}'
          footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-pm-time:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    name: Report duration per package manager
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: 'skipped'
          message_format: '${{ needs.process-result.outputs.pm-duration }}'
          notification_title: 'Total duration per package manager (ubuntu only)'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

  report-proj-time:
    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
    needs: process-result
    runs-on: ubuntu-latest
    name: Report duration per package manager
    steps:
      - name: Send notification
        uses: ravsamhq/notify-slack-action@v2
        with:
          status: 'skipped'
          message_format: '${{ needs.process-result.outputs.proj-duration }}'
          notification_title: 'E2E Project duration stats'
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}