wip: set up sanity docker check with composites

dgrebb · Jan 16, 2024 · bf49750 · bf49750
1 parent 90fcc2f
commit bf49750
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 121 deletions.
diff --git a/...actions/prepare-docker/prepare-docker.yml → .github/actions/prepare-docker/action.yml b/...actions/prepare-docker/prepare-docker.yml → .github/actions/prepare-docker/action.yml
@@ -5,6 +5,8 @@ runs:
   using: "composite"
   steps:
     - name: Set Name and Tag Vars
+      shell: bash
+      working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
       env:
         name: "${{ env.BRANCH_NAME }}"
       run: |
@@ -21,5 +23,7 @@ runs:
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Pull Image
+      shell: bash
+      working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
       run: |
         docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
diff --git a/.github/workflows/docker-sanity-test.yml b/.github/workflows/docker-sanity-test.yml
@@ -27,81 +27,33 @@ jobs:
       packages: write
 
     steps:
-      - uses: actions/checkout
-      - name: Checkout
+      - name: Checkout actions
         uses: actions/checkout@v4
         with:
-          fetch-depth: 1
-          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+          ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
+          sparse-checkout: |
+            .github/actions
+          path: actions
 
       - name: "Setup Node, npm Cache, and Install"
         uses: ./.github/actions/setup-base
 
-      - name: Set Name and Tag Vars
-        env:
-          name: "${{ env.BRANCH_NAME }}"
-        run: |
-          echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV}
-          echo "TAG=${name/\//-}" >> $GITHUB_ENV
-          echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
-          echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')"  >> $GITHUB_ENV
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Pull Image
-        run: |
-          docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
+      - uses: ./.github/actions/prepare-docker
 
       - name: "𓋏 Run `backstop test` in Docker"
         continue-on-error: true
+        shell: bash
+        working-directory: ${{ steps.base.outputs.WORKSPACE_ROOT }}
         run: |
           set +e
           cd test/configs/ && docker run --rm -t --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG test
 
       - name: "Validate Puppeteer Docker Test Results"
-        continue-on-error: true
-        run: |
-          set +e
-          PUPPETEER_TEST_RESULT=$(diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-docker.json) \
-            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json))
-          echo "PUPPETEER_TEST_RESULT=$PUPPETEER_TEST_RESULT" >> $GITHUB_ENV
-          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
-            echo "# ❎ Puppeteer Smoke Different" >> $GITHUB_STEP_SUMMARY
-            echo '```diff' >> $GITHUB_STEP_SUMMARY
-            echo "${PUPPETEER_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
-            echo '```' >> $GITHUB_STEP_SUMMARY
-          else
-            echo "# ✅ Puppeteer Smoke Report Valid" >> $GITHUB_STEP_SUMMARY
-          fi
-
-      - name: "Full Smoke Report Diff"
-        id: diff
-        continue-on-error: true
-        run: |
-          set +e
-          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
-          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
-          echo "<details>" >> $GITHUB_STEP_SUMMARY
-          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo '```diff' >> $GITHUB_STEP_SUMMARY
-          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
-          echo '```' >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "</details>" >> $GITHUB_STEP_SUMMARY
-
-      - name: "Report Validation Outcome"
-        run: |
-          if [[ "$PUPPETEER_TEST_RESULT" != "" ]]; then
-            exit 1
-          else
-            exit 0
-          fi
+        uses: ./actions/.github/actions/sanity-test-checks
+        with:
+          WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+          RUNNER: "Puppeteer"
+          FIXTURE: "sanity-test-docker.json"
 
   sanity-test-playwright:
     name: 🤪 Playwright
@@ -112,34 +64,18 @@ jobs:
       packages: write
 
     steps:
-      - name: Checkout
+      - name: Checkout actions
         uses: actions/checkout@v4
         with:
-          fetch-depth: 1
-          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+          ref: ${{ github.event.pull_request.head.sha || github.head_ref || github.ref_name || github.ref }}
+          sparse-checkout: |
+            .github/actions
+          path: actions
 
       - name: "Setup Node, npm Cache, and Install"
         uses: ./.github/actions/setup-base
 
-      - name: Set Name and Tag Vars
-        env:
-          name: "${{ env.BRANCH_NAME }}"
-        run: |
-          echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV}
-          echo "TAG=${name/\//-}" >> $GITHUB_ENV
-          echo "PV=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV
-          echo "PLAYWRIGHT_VERSION=$(cat package.json | jq -r '.dependencies.playwright')"  >> $GITHUB_ENV
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Pull Image
-        run: |
-          docker pull $REGISTRY/$IMAGE_NAME_LC:$TAG
+      - uses: ./.github/actions/prepare-docker
 
       - name: "🎭 Run `backstop test --confg=playwright` in Docker"
         continue-on-error: true
@@ -148,41 +84,8 @@ jobs:
           cd test/configs/ && docker run --rm -t --entrypoint='' --mount type=bind,source="$(pwd)",target=/src $REGISTRY/$IMAGE_NAME_LC:$TAG sh -c "chmod -R 777 /root && chmod -R 777 /opt/pw-browsers && npm --verbose --foreground-scripts i -D playwright && npx --verbose --foreground-scripts --yes playwright@$PLAYWRIGHT_VERSION install && backstop test --config=playwright"
 
       - name: "Validate Playwright Docker Test Results"
-        continue-on-error: true
-        run: |
-          set +e
-          diff -c <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/__fixtures__/sanity-test-playwright-docker.json) \
-            <(jq 'walk(if type == "object" then with_entries(.value |= if type == "object" or type == "array" then . else "" end) else . end) | del(.tests[].pair.diff)' test/configs/backstop_data/bitmaps_test/**/report.json)
-          echo "PLAYWRIGHT_TEST_RESULT=$PLAYWRIGHT_TEST_RESULT" >> $GITHUB_ENV
-          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
-            echo "# ❎ Playwright Sanity Report Different" >> $GITHUB_STEP_SUMMARY
-            echo '```diff' >> $GITHUB_STEP_SUMMARY
-            echo "${PLAYWRIGHT_TEST_RESULT}" >> $GITHUB_STEP_SUMMARY
-            echo '```' >> $GITHUB_STEP_SUMMARY
-          else
-            echo "# ✅ Playwright Sanity Report Valid" >> $GITHUB_STEP_SUMMARY
-          fi
-
-      - name: "Full Sanity Report Diff"
-        id: diff
-        continue-on-error: true
-        run: |
-          set +e
-          FULL_TEST_DIFF=$(diff <(jq -S '.tests[]' test/__fixtures__/sanity-test-playwright-docker.json) <(jq -S '.tests[]' test/configs/backstop_data/bitmaps_test/**/report.json))
-          echo "## Unfiltered Diff" >> $GITHUB_STEP_SUMMARY
-          echo "<details>" >> $GITHUB_STEP_SUMMARY
-          echo "<summary>Expand Diff</summary>" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo '```diff' >> $GITHUB_STEP_SUMMARY
-          echo "${FULL_TEST_DIFF}" >> $GITHUB_STEP_SUMMARY
-          echo '```' >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo "</details>" >> $GITHUB_STEP_SUMMARY
-
-      - name: "Report Validation Outcome"
-        run: |
-          if [[ "$PLAYWRIGHT_TEST_RESULT" != "" ]]; then
-            exit 1
-          else
-            exit 0
-          fi
+        uses: ./actions/.github/actions/sanity-test-checks
+        with:
+          WORKSPACE_ROOT: ${{ steps.base.outputs.WORKSPACE_ROOT }}
+          RUNNER: "Playwright"
+          FIXTURE: "sanity-test-playwright-docker.json"