From 23261daf3a250d6cf4262f7149a7a03b846a0fc3 Mon Sep 17 00:00:00 2001 From: Dan Grebb Date: Tue, 16 Jan 2024 17:19:05 -0500 Subject: [PATCH] feat(ci): refactor test checks with composite actions (garris#1533) --- .github/actions/cache-playwright/action.yml | 38 ++++ .github/actions/prepare-docker/action.yml | 40 +++++ .github/actions/sanity-test-checks/action.yml | 86 +++++++++ .github/actions/setup-base/action.yml | 23 +++ .github/actions/setup-node/action.yml | 22 +++ .github/actions/smoke-test-checks/action.yml | 93 ++++++++++ .../workflows/docker-sanity-test-checks.yml | 168 ++++-------------- .../workflows/docker-smoke-test-checks.yml | 168 ++++-------------- .github/workflows/integration-test-check.yml | 24 +-- .github/workflows/sanity-test-checks.yml | 140 +++++---------- .github/workflows/smoke-test-checks.yml | 136 ++++---------- 11 files changed, 460 insertions(+), 478 deletions(-) create mode 100644 .github/actions/cache-playwright/action.yml create mode 100644 .github/actions/prepare-docker/action.yml create mode 100644 .github/actions/sanity-test-checks/action.yml create mode 100644 .github/actions/setup-base/action.yml create mode 100644 .github/actions/setup-node/action.yml create mode 100644 .github/actions/smoke-test-checks/action.yml diff --git a/.github/actions/cache-playwright/action.yml b/.github/actions/cache-playwright/action.yml new file mode 100644 index 000000000..11235f259 --- /dev/null +++ b/.github/actions/cache-playwright/action.yml @@ -0,0 +1,38 @@ +name: "Install and Cache Playwright" +description: "Sets up a cache and/or installs Playwright and its binaries." + +inputs: + WORKSPACE_ROOT: + description: "The workspace root." + required: true + +runs: + using: "composite" + steps: + - name: Get installed Playwright version + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + id: playwright-version + run: echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright' || 'latest')" >> $GITHUB_ENV + + - name: Cache playwright binaries + uses: actions/cache@v3 + id: playwright-cache + with: + path: | + ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }} + + - shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: npm ci + + - shell: bash + if: steps.playwright-cache.outputs.cache-hit != 'true' + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: npx playwright install --with-deps + + - shell: bash + if: steps.playwright-cache.outputs.cache-hit != 'true' + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: npx playwright install-deps diff --git a/.github/actions/prepare-docker/action.yml b/.github/actions/prepare-docker/action.yml new file mode 100644 index 000000000..5dad1048d --- /dev/null +++ b/.github/actions/prepare-docker/action.yml @@ -0,0 +1,40 @@ +name: "Pull & Prepare Docker Workflow" +description: "Sets name and tag, logs in to GHCR, and pulls Docker image." + +inputs: + REGISTRY: + description: "GitHub Registry" + required: true + ACTOR: + description: "GitHub Username" + required: true + GITHUB_TOKEN: + description: "GitHub Token" + required: true + +runs: + using: "composite" + steps: + - name: Set Name and Tag Vars + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} + env: + name: "${{ env.BRANCH_NAME }}" + run: | + echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV} + echo "TAG=${name/\//-}" >> $GITHUB_ENV + echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV + echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ inputs.REGISTRY }} + username: ${{ inputs.ACTOR }} + password: ${{ inputs.GITHUB_TOKEN }} + + - name: Pull Image + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} + run: | + docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG diff --git a/.github/actions/sanity-test-checks/action.yml b/.github/actions/sanity-test-checks/action.yml new file mode 100644 index 000000000..b0db609a6 --- /dev/null +++ b/.github/actions/sanity-test-checks/action.yml @@ -0,0 +1,86 @@ +name: "Validate Sanity Report" +description: "Diff the sanity-test reports with a fixture and validate expected shape." + +inputs: + WORKSPACE_ROOT: + description: "Working Directory" + required: true + RUNNER: + description: "Test Runner" + required: true + FIXTURE: + description: "Test Report Fixture" + required: true + +runs: + using: "composite" + steps: + - name: "Validate ${{ inputs.RUNNER }} Report" + id: validate-report + continue-on-error: true + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: | + set +e + FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}" + TEST_RESULT=$(\ + diff -c \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" \ + then . \ + else \ + "" \ + end) \ + else . \ + end)' \ + $FIXTURE) \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" \ + then . \ + else \ + "" \ + end) \ + else . \ + end)' \ + test/configs/backstop_data/bitmaps_test/**/report.json) \ + ) + + echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV + if [[ "$TEST_RESULT" != "" ]]; then + echo "# ❎ ${{ inputs.RUNNER }} Sanity Different" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo "${TEST_RESULT}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + else + echo "# ✅ ${{ inputs.RUNNER }} Sanity Report Valid" >> $GITHUB_STEP_SUMMARY + fi + + - name: "Full Sanity Report Diff" + id: diff + continue-on-error: true + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: | + set +e + FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' $FIXTURE) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "Expand Diff" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + + - name: "Report Validation Outcome" + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: | + if [[ "$TEST_RESULT" != "" ]]; then + exit 1 + else + exit 0 + fi diff --git a/.github/actions/setup-base/action.yml b/.github/actions/setup-base/action.yml new file mode 100644 index 000000000..ac5a56a67 --- /dev/null +++ b/.github/actions/setup-base/action.yml @@ -0,0 +1,23 @@ +name: "Setup Base Workflow" +description: "Checks out the BackstopJS repo at the calling or current branch with shallow history. Pulls Docker image by branch tag. Installs dependencies. Optionally sets up interactive remote debug shell." + +outputs: + WORKSPACE_ROOT: + description: "Export the root workspace for all workflows." + value: ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} + +runs: + using: "composite" + steps: + - name: Set workspace root + id: workspace_root + shell: bash + run: | + WORKSPACE_ROOT=$GITHUB_WORKSPACE/repo + echo "WORKSPACE_ROOT=$WORKSPACE_ROOT" | tee -a $GITHUB_OUTPUT + + - name: Checkout to workspace + shell: bash + run: | + git clone -b ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} https://github.com/${{ github.repository }}.git ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} + ls ${{ steps.workspace_root.outputs.WORKSPACE_ROOT }} diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml new file mode 100644 index 000000000..271694941 --- /dev/null +++ b/.github/actions/setup-node/action.yml @@ -0,0 +1,22 @@ +name: "Setup Node and Dependencies" +description: "Sets up Node, npm caching, and installs dependencies" + +inputs: + WORKSPACE_ROOT: + description: "The workspace root." + required: true + +runs: + using: "composite" + steps: + - name: ⬢ Setup Node & Cache + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: "npm" + cache-dependency-path: ${{ inputs.WORKSPACE_ROOT }}/package-lock.json + + - name: ↧ Install + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: npm ci diff --git a/.github/actions/smoke-test-checks/action.yml b/.github/actions/smoke-test-checks/action.yml new file mode 100644 index 000000000..aa9b5739e --- /dev/null +++ b/.github/actions/smoke-test-checks/action.yml @@ -0,0 +1,93 @@ +name: "Validate Smoke Report" +description: "Diff the smoke-test reports with a fixture and validate expected shape." + +inputs: + WORKSPACE_ROOT: + description: "Working Directory" + required: true + RUNNER: + description: "Test Runner" + required: true + FIXTURE: + description: "Test Report Fixture" + required: true + +runs: + using: "composite" + steps: + - name: "Validate ${{ inputs.RUNNER }} Report" + id: validate + continue-on-error: true + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: | + set +e + FIXTURE="test/__fixtures__/${{ inputs.FIXTURE }}" + TEST_RESULT=$(\ + diff -c \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" + then . \ + else \ + "" \ + end) \ + else . \ + end) | \ + del(\ + .tests[].pair.diff, \ + .tests[].pair.diffImage \ + )'\ + $FIXTURE) \ + <(jq \ + 'walk(if type == "object" then with_entries(.value |= \ + if type == "object" or type == "array" \ + then . \ + else \ + "" \ + end) \ + else . \ + end) | \ + del(\ + .tests[].pair.diff, \ + .tests[].pair.diffImage\ + )' \ + test/configs/backstop_data/bitmaps_test/**/report.json)) + + echo "TEST_RESULT=$TEST_RESULT" >> $GITHUB_ENV + if [[ "$TEST_RESULT" != "" ]]; then + echo "# ❎ ${{ inputs.RUNNER }} Report Different" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo "$TEST_RESULT" >> $GITHUB_STEP_SUMMARY + echo '```' + else + echo "# ✅ ${{ inputs.RUNNER }} Report Validated" >> $GITHUB_STEP_SUMMARY + fi + + - name: "Full Smoke Report Diff" + id: diff + continue-on-error: true + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: | + set +e + FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' $FIXTURE) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) + echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "Expand Diff" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + + - name: "Report Validation Outcome" + shell: bash + working-directory: ${{ inputs.WORKSPACE_ROOT }} + run: | + if [[ "$TEST_RESULT" != "" ]]; then + exit 1 + else + exit 0 + fi diff --git a/.github/workflows/docker-sanity-test-checks.yml b/.github/workflows/docker-sanity-test-checks.yml index b40dba812..55e8c5e52 100644 --- a/.github/workflows/docker-sanity-test-checks.yml +++ b/.github/workflows/docker-sanity-test-checks.yml @@ -27,86 +27,37 @@ jobs: packages: write steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: Set Name and Tag Vars - env: - name: "${{ env.BRANCH_NAME }}" - run: | - echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV} - echo "TAG=${name/\//-}" >> $GITHUB_ENV - echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV - echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - uses: ./actions/.github/actions/setup-base + id: base - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 + - uses: ./actions/.github/actions/prepare-docker with: - cache: "npm" - cache-dependency-path: package-lock.json - - - name: ↧ Install - run: npm ci - - - name: Pull Image - run: | - docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG + REGISTRY: ${{ env.REGISTRY }} + ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "𓋏 Run `backstop test` in Docker" continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} run: | set +e cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test - name: "Validate Puppeteer Docker Test Results" - continue-on-error: true - run: | - set +e - PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-docker.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Smoke Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + uses: ./actions/.github/actions/sanity-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Puppeteer" + FIXTURE: "sanity-test-docker.json" sanity-test-playwright: name: 🤪 Playwright @@ -117,40 +68,22 @@ jobs: packages: write steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: Set Name and Tag Vars - env: - name: "${{ env.BRANCH_NAME }}" - run: | - echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV} - echo "TAG=${name/\//-}" >> $GITHUB_ENV - echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV - echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')" >> $GITHUB_ENV - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - uses: ./actions/.github/actions/setup-base + id: base - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 + - uses: ./actions/.github/actions/prepare-docker with: - cache: "npm" - cache-dependency-path: package-lock.json - - - name: ↧ Install - run: npm ci --verbose --foreground-scripts - - - name: Pull Image - run: | - docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG + REGISTRY: ${{ env.REGISTRY }} + ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "🎭 Run `backstop test --confg=playwright` in Docker" continue-on-error: true @@ -159,41 +92,8 @@ jobs: cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose --foreground-scripts i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=playwright" - name: "Validate Playwright Docker Test Results" - continue-on-error: true - run: | - set +e - diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-playwright-docker.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json) - echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Sanity Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + uses: ./actions/.github/actions/sanity-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Playwright" + FIXTURE: "sanity-test-playwright-docker.json" diff --git a/.github/workflows/docker-smoke-test-checks.yml b/.github/workflows/docker-smoke-test-checks.yml index 56be978b5..a0b3998a7 100644 --- a/.github/workflows/docker-smoke-test-checks.yml +++ b/.github/workflows/docker-smoke-test-checks.yml @@ -27,39 +27,22 @@ jobs: packages: write steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: Set Name and Tag Vars - env: - name: "${{ env.BRANCH_NAME }}" - run: | - echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV} - echo "TAG=${name/\//-}" >> $GITHUB_ENV - echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - uses: ./actions/.github/actions/setup-base + id: base - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 + - uses: ./actions/.github/actions/prepare-docker with: - cache: "npm" - cache-dependency-path: package-lock.json - - - name: ↧ Install - run: npm ci - - - name: Pull Image - run: | - docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG + REGISTRY: ${{ env.REGISTRY }} + ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "𓋏 Run `backstop test --confg=backstop_features` in Docker" continue-on-error: true @@ -67,45 +50,12 @@ jobs: set +e cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test --config=backstop_features - - name: "Validate Puppeteer Report" - continue-on-error: true - run: | - set +e - PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/smoke-test-docker.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Smoke Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/smoke-test-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - name: "Validate Puppeteer Docker Test Results" + uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Puppeteer" + FIXTURE: "smoke-test-docker.json" smoke-test-playwright: name: 💨 Playwright @@ -116,39 +66,22 @@ jobs: packages: write steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: Set Name and Tag Vars - env: - name: "${{ env.BRANCH_NAME }}" - run: | - echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV} - echo "TAG=${name/\//-}" >> $GITHUB_ENV - echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - uses: ./actions/.github/actions/setup-base + id: base - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 + - uses: ./actions/.github/actions/prepare-docker with: - cache: "npm" - cache-dependency-path: package-lock.json - - - name: ↧ Install - run: npm ci - - - name: Pull Image - run: | - docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG + REGISTRY: ${{ env.REGISTRY }} + ACTOR: ${{ github.actor }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "🎭 Run `backstop test --confg=backstop_features_pw` in Docker" continue-on-error: true @@ -156,42 +89,9 @@ jobs: set +e cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test --config=backstop_features_pw - - name: "Validate Playwright Report" - continue-on-error: true - run: | - set +e - PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/smoke-test-playwright-docker.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Smoke Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/smoke-test-playwright-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - name: "Validate Playwright Docker Test Results" + uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Playwright" + FIXTURE: "smoke-test-playwright-docker.json" diff --git a/.github/workflows/integration-test-check.yml b/.github/workflows/integration-test-check.yml index c903e7776..c90e434e2 100644 --- a/.github/workflows/integration-test-check.yml +++ b/.github/workflows/integration-test-check.yml @@ -17,22 +17,26 @@ jobs: name: 🧩 Validate Integration Test Results runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 - with: - cache: "npm" - cache-dependency-path: package-lock.json + - uses: ./actions/.github/actions/setup-base + id: base - - name: ↧ Install - run: npm ci + - uses: ./actions/.github/actions/setup-node + id: node + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} - name: "𓋏 Run `npm run integration-test`" + continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} run: | npm run integration-test diff --git a/.github/workflows/sanity-test-checks.yml b/.github/workflows/sanity-test-checks.yml index 6b0b3519e..c2b89579e 100644 --- a/.github/workflows/sanity-test-checks.yml +++ b/.github/workflows/sanity-test-checks.yml @@ -2,6 +2,7 @@ name: Sanity Test Checks on: workflow_dispatch: + workflow_call: permissions: @@ -17,129 +18,68 @@ jobs: name: 🤪 Puppeteer runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 - with: - cache: "npm" - cache-dependency-path: package-lock.json + - uses: ./actions/.github/actions/setup-base + id: base - - name: ↧ Install - run: npm ci + - uses: ./actions/.github/actions/setup-node + id: node + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} - name: "𓋏 Run `npm run sanity-test`" continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} run: | npm run sanity-test - - name: "Validate Puppeteer Report" - id: validate-puppeteer - continue-on-error: true - run: | - set +e - PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Sanity Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - uses: ./actions/.github/actions/sanity-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Puppeteer" + FIXTURE: "sanity-test.json" sanity-playwright: name: 🤪 Playwright runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 + - uses: ./actions/.github/actions/setup-base + id: base + + - uses: ./actions/.github/actions/setup-node + id: node with: - cache: "npm" - cache-dependency-path: package-lock.json + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} - - name: ↧ Install - run: npm ci + - uses: ./actions/.github/actions/cache-playwright + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} - name: "🎭 Run `npm run sanity-test-playwright`" continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} run: | - npx playwright install --with-deps npm run sanity-test-playwright - - name: "Validate Playwright Report" - id: validate-playwright - continue-on-error: true - run: | - set +e - PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/__fixtures__/sanity-test-playwright.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Sanity Report Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - uses: ./actions/.github/actions/sanity-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Playwright" + FIXTURE: "sanity-test-playwright.json" diff --git a/.github/workflows/smoke-test-checks.yml b/.github/workflows/smoke-test-checks.yml index c98ad98d8..c9d66d62d 100644 --- a/.github/workflows/smoke-test-checks.yml +++ b/.github/workflows/smoke-test-checks.yml @@ -17,129 +17,65 @@ jobs: name: 💨 Puppeteer runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 - with: - cache: "npm" - cache-dependency-path: package-lock.json + - uses: ./actions/.github/actions/setup-base + id: base - - name: ↧ Install - run: npm ci + - uses: ./actions/.github/actions/setup-node + id: node + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} - name: "𓋏 Run `npm run smoke-test`" continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} run: | npm run smoke-test - - name: "Validate Puppeteer Report" - id: validate - continue-on-error: true - run: | - set +e - PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test.json) \ - <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "$PUPPETEER_TEST_RESULT" >> $GITHUB_STEP_SUMMARY - echo '```' - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Smoke Test Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Puppeteer" + FIXTURE: "smoke-test.json" smoke-playwright: name: 💨 Playwright runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout actions uses: actions/checkout@v4 with: - fetch-depth: 1 - ref: ${{ github.event.pull_request.head.sha || github.ref }} + ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }} + sparse-checkout: | + .github/actions + path: actions - - name: ⬢ Setup Node & Cache - uses: actions/setup-node@v4 - with: - cache: "npm" - cache-dependency-path: package-lock.json + - uses: ./actions/.github/actions/setup-base + id: base - - name: ↧ Install - run: npm ci + - uses: ./actions/.github/actions/setup-node + id: node + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} - name: "🎭 Run `npm run smoke-test-playwright`" continue-on-error: true + shell: bash + working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }} run: | npx playwright install --with-deps npm run smoke-test-playwright - - name: "Validate Playwright Report" - id: validate - continue-on-error: true - run: | - set +e - PLAYWRIGHT_TEST_RESULT=$(diff -c <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/__fixtures__/smoke-test-playwright.json) \ - <(jq -S 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end)' | del(.tests[].pair.diff, .tests[].pair.diffImage)' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - echo "# ❎ Report Different" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_STEP_SUMMARY - echo '```' - else - echo "# ✅ Report Validated" >> $GITHUB_STEP_SUMMARY - fi - - - name: "Full Smoke Test Diff" - id: diff - continue-on-error: true - run: | - set +e - FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json)) - echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - echo "Expand Diff" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "
" >> $GITHUB_STEP_SUMMARY - - - name: "Report Validation Outcome" - run: | - if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then - exit 1 - else - exit 0 - fi + - uses: ./actions/.github/actions/smoke-test-checks + with: + WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }} + RUNNER: "Playwright" + FIXTURE: "smoke-test-playwright.json"