Backstop Sanity Tests #30
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sanity Test Checks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| DEBUG_ENABLED: | |
| description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)" | |
| type: boolean | |
| required: false | |
| default: false | |
| DEBUG_DETACHED: | |
| description: "whether to launch tmate in detached mode (if the tmate action is run)" | |
| type: boolean | |
| required: false | |
| default: true | |
| workflow_call: | |
| permissions: | |
| actions: write | |
| contents: write | |
| pull-requests: write | |
| env: | |
| NODE_VERSION: 20 | |
| jobs: | |
| sanity-puppeteer: | |
| name: 🤪 Puppeteer | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout actions | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} | |
| sparse-checkout: | | |
| .github/actions | |
| path: actions | |
| - uses: ./actions/.github/actions/setup-base | |
| id: base | |
| with: | |
| DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }} | |
| DEBUG_DETACHED: ${{ inputs.DEBUG_DETACHED }} | |
| - name: Checkout to workspace | |
| shell: bash | |
| run: | | |
| rm -rf ${{ steps.base.outputs.WORKSPACE_ROOT }}/* | |
| rm -rf ${{ steps.base.outputs.WORKSPACE_ROOT }}/*.* | |
| git clone -b ${{ github.head_ref || github.ref_name }} https://github.com/${{ github.repository }}.git ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| ls ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| - name: "𓋏 Run `npm run sanity-test`" | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| npm run sanity-test | |
| - name: "Validate Puppeteer Report" | |
| id: validate-puppeteer | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test.json) \ | |
| <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV | |
| if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then | |
| echo "# ❎ Puppeteer Sanity Different" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "# ✅ Puppeteer Sanity Report Valid" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: "Full Sanity Report Diff" | |
| id: diff | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY | |
| echo "<details>" >> $GITHUB_STEP_SUMMARY | |
| echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "</details>" >> $GITHUB_STEP_SUMMARY | |
| - name: "Report Validation Outcome" | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then | |
| exit 1 | |
| else | |
| exit 0 | |
| fi | |
| sanity-playwright: | |
| name: 🤪 Playwright | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout actions | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} | |
| sparse-checkout: | | |
| .github/actions | |
| path: actions | |
| - uses: ./actions/.github/actions/setup-base | |
| id: base | |
| - name: "🎭 Run `npm run sanity-test-playwright`" | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| npx playwright install --with-deps | |
| npm run sanity-test-playwright | |
| - name: "Validate Playwright Report" | |
| id: validate-playwright | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test-playwright.json) \ | |
| <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV | |
| if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then | |
| echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: "Full Sanity Report Diff" | |
| id: diff | |
| continue-on-error: true | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| set +e | |
| FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) | |
| echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY | |
| echo "<details>" >> $GITHUB_STEP_SUMMARY | |
| echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '```diff' >> $GITHUB_STEP_SUMMARY | |
| echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "</details>" >> $GITHUB_STEP_SUMMARY | |
| - name: "Report Validation Outcome" | |
| shell: bash | |
| working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} | |
| run: | | |
| if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then | |
| exit 1 | |
| else | |
| exit 0 | |
| fi |