backstop-smoke-test.yml

name: Smoke Test Checks

on:
  workflow_dispatch:
  workflow_call:

permissions:
  actions: write
  contents: write
  pull-requests: write

env:
  NODE_VERSION: 20

jobs:
  smoke-puppeteer:
    name: 💨 Puppeteer
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha || github.ref }}

      - name: "Setup Node, npm Cache, and Install"
        uses: ./.github/actions/setup-base

      - name: "𓋏 Run `npm run smoke-test`"
        continue-on-error: true
        run: |
          npm run smoke-test

      - name: "Validate Puppeteer Report"
        id: validate
        continue-on-error: true
        run: |
          set +e
          PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test.json) \
            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
            echo "# ❎ Puppeteer Report Different" >> $GITHUB_STEP_SUMMARY
            echo '```diff' >> $GITHUB_STEP_SUMMARY
            echo "$PUPPETEER_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
            echo '```'
          else
            echo "# ✅ Puppeteer Report Validated" >> $GITHUB_STEP_SUMMARY
          fi

      - name: "Full Smoke Test Diff"
        id: diff
        continue-on-error: true
        run: |
          set +e
          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
          echo "<details>" >> $GITHUB_STEP_SUMMARY
          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo '```diff' >> $GITHUB_STEP_SUMMARY
          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "</details>" >> $GITHUB_STEP_SUMMARY

      - name: "Report Validation Outcome"
        run: |
          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
            exit 1
          else
            exit 0
          fi

  smoke-playwright:
    name: 💨 Playwright
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          ref: ${{ github.event.pull_request.head.sha || github.ref }}
            - name: "Setup Node, npm Cache, and Install"
        uses: ./.github/actions/setup-base

      - name: "🎭 Run `npm run smoke-test-playwright`"
        continue-on-error: true
        run: |
          npx playwright install --with-deps
          npm run smoke-test-playwright

      - name: "Validate Playwright Report"
        id: validate
        continue-on-error: true
        run: |
          set +e
          PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test-playwright.json) \
            <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
            echo "# ❎ Playwright Report Different" >> $GITHUB_STEP_SUMMARY
            echo '```diff' >> $GITHUB_STEP_SUMMARY
            echo "$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_STEP_SUMMARY
            echo '```'
          else
            echo "# ✅ Playwright Report Validated" >> $GITHUB_STEP_SUMMARY
          fi

      - name: "Full Smoke Test Diff"
        id: diff
        continue-on-error: true
        run: |
          set +e
          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
          echo "<details>" >> $GITHUB_STEP_SUMMARY
          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo '```diff' >> $GITHUB_STEP_SUMMARY
          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "</details>" >> $GITHUB_STEP_SUMMARY

      - name: "Report Validation Outcome"
        run: |
          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
            exit 1
          else
            exit 0
          fi