Backstop Sanity Tests #43
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sanity Test Checks | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| permissions: | |
| actions: write | |
| contents: write | |
| pull-requests: write | |
| env: | |
| NODE_VERSION: 20 | |
| jobs: | |
| sanity-puppeteer: | |
| name: 🤪 Puppeteer | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout actions | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} | |
| sparse-checkout: | | |
| .github/actions | |
| path: actions | |
| - uses: ./actions/.github/actions/setup-base | |
| id: base | |
| - uses: ./actions/.github/actions/setup-node | |
| id: node | |
| with: | |
| WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| - name: "𓋏 Run `npm run sanity-test`" | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| npm run sanity-test | |
| - name: "Validate Puppeteer Report" | |
| id: validate-puppeteer | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test.json) \ | |
| <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV | |
| if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then | |
| echo "# ❎ Puppeteer Sanity Different" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "# ✅ Puppeteer Sanity Report Valid" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: "Full Sanity Report Diff" | |
| id: diff | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY | |
| echo "<details>" >> $GITHUB_STEP_SUMMARY | |
| echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "</details>" >> $GITHUB_STEP_SUMMARY | |
| - name: "Report Validation Outcome" | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then | |
| exit 1 | |
| else | |
| exit 0 | |
| fi | |
| sanity-playwright: | |
| name: 🤪 Playwright | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout actions | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} | |
| sparse-checkout: | | |
| .github/actions | |
| path: actions | |
| - uses: ./actions/.github/actions/setup-base | |
| id: base | |
| - uses: ./actions/.github/actions/setup-node | |
| id: node | |
| with: | |
| WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| - name: "🎭 Run `npm run sanity-test-playwright`" | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| npx playwright install --with-deps | |
| npm run sanity-test-playwright | |
| - name: "Validate Playwright Report" | |
| id: validate-playwright | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test-playwright.json) \ | |
| <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV | |
| if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then | |
| echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: "Full Sanity Report Diff" | |
| id: diff | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY | |
| echo "<details>" >> $GITHUB_STEP_SUMMARY | |
| echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "</details>" >> $GITHUB_STEP_SUMMARY | |
| - name: "Report Validation Outcome" | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then | |
| exit 1 | |
| else | |
| exit 0 | |
| fi |