Skip to content

Commit

Permalink
wip: set up sanity docker check with composites
Browse files Browse the repository at this point in the history
  • Loading branch information
dgrebb committed Jan 16, 2024
1 parent 90fcc2f commit 68f3d76
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 114 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ runs:
using: "composite"
steps:
- name: Set Name and Tag Vars
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
env:
name: "${{ env.BRANCH_NAME }}"
run: |
Expand All @@ -21,5 +23,7 @@ runs:
password: ${{ secrets.GITHUB_TOKEN }}

- name: Pull Image
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
128 changes: 14 additions & 114 deletions .github/workflows/docker-sanity-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,71 +37,22 @@ jobs:
- name: "Setup Node, npm Cache, and Install"
uses: ./.github/actions/setup-base

- name: Set Name and Tag Vars
env:
name: "${{ env.BRANCH_NAME }}"
run: |
echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV}
echo "TAG=${name/\//-}" >> $GITHUB_ENV
echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Pull Image
run: |
docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
- uses: ./.github/actions/prepare-docker

- name: "𓋏 Run `backstop test` in Docker"
continue-on-error: true
shell: bash
working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
run: |
set +e
cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test
- name: "Validate Puppeteer Docker Test Results"
continue-on-error: true
run: |
set +e
PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-docker.json) \
<(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
echo "# ❎ Puppeteer Smoke Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
else
echo "# ✅ Puppeteer Smoke Report Valid" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Smoke Report Diff"
id: diff
continue-on-error: true
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
run: |
if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi
uses: ./actions/.github/actions/sanity-test-checks
with:
WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
RUNNER: "Puppeteer"
FIXTURE: "sanity-test-docker.json"

sanity-test-playwright:
name: 🤪 Playwright
Expand All @@ -121,25 +72,7 @@ jobs:
- name: "Setup Node, npm Cache, and Install"
uses: ./.github/actions/setup-base

- name: Set Name and Tag Vars
env:
name: "${{ env.BRANCH_NAME }}"
run: |
echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV}
echo "TAG=${name/\//-}" >> $GITHUB_ENV
echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Pull Image
run: |
docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
- uses: ./.github/actions/prepare-docker

- name: "🎭 Run `backstop test --confg=playwright` in Docker"
continue-on-error: true
Expand All @@ -148,41 +81,8 @@ jobs:
cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose --foreground-scripts i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=playwright"
- name: "Validate Playwright Docker Test Results"
continue-on-error: true
run: |
set +e
diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-playwright-docker.json) \
<(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json)
echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
else
echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
fi
- name: "Full Sanity Report Diff"
id: diff
continue-on-error: true
run: |
set +e
FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```diff' >> $GITHUB_STEP_SUMMARY
echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
- name: "Report Validation Outcome"
run: |
if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
exit 1
else
exit 0
fi
uses: ./actions/.github/actions/sanity-test-checks
with:
WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
RUNNER: "Playwright"
FIXTURE: "sanity-test-playwright-docker.json"

0 comments on commit 68f3d76

Please sign in to comment.