diff --git a/.github/actions/e2e-pw/run-changed-tests/action.yml b/.github/actions/e2e-pw/run-changed-tests/action.yml new file mode 100644 index 00000000000..d0be312794a --- /dev/null +++ b/.github/actions/e2e-pw/run-changed-tests/action.yml @@ -0,0 +1,17 @@ +name: 'Run Changed Tests' +description: 'Runs Playwright E2E Only Changed Tests' + +runs: + using: 'composite' + steps: + - name: First Run Playwright E2E Tests (Only Changed) + id: first_run_e2e_pw_tests_only_changed + # Use +e to trap errors when running E2E tests. + shell: /bin/bash +e {0} + run: npm run test:e2e-pw-ci-only-changed + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: playwright-report/ + retention-days: 7 diff --git a/.github/workflows/e2e-pw-pull-request-only-changed.yml b/.github/workflows/e2e-pw-pull-request-only-changed.yml new file mode 100644 index 00000000000..f1675cecfef --- /dev/null +++ b/.github/workflows/e2e-pw-pull-request-only-changed.yml @@ -0,0 +1,75 @@ +name: E2E Playwright Tests - Pull Request - Only Changed + +on: + pull_request: + branches: + - develop + - trunk + workflow_dispatch: + workflow_call: + inputs: + wcpay-use-build-artifact: + type: boolean + required: false + default: false + repo-branch: + type: string + required: false + description: 'Branch to be used for running tests' + +env: + E2E_GH_TOKEN: ${{ secrets.E2E_GH_TOKEN }} + WCP_DEV_TOOLS_REPO: ${{ secrets.WCP_DEV_TOOLS_REPO }} + WCP_DEV_TOOLS_BRANCH: 'trunk' + TRANSACT_PLATFORM_SERVER_REPO: ${{ secrets.TRANSACT_PLATFORM_SERVER_REPO }} + WC_SUBSCRIPTIONS_REPO: ${{ secrets.WC_SUBSCRIPTIONS_REPO }} + E2E_BLOG_ID: ${{ secrets.E2E_BLOG_ID }} + E2E_BLOG_TOKEN: ${{ secrets.E2E_BLOG_TOKEN }} + E2E_USER_TOKEN: ${{ secrets.E2E_USER_TOKEN }} + WC_E2E_SCREENSHOTS: 1 + E2E_SLACK_CHANNEL: ${{ secrets.E2E_SLACK_CHANNEL }} + E2E_SLACK_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }} + E2E_USE_LOCAL_SERVER: false + E2E_RESULT_FILEPATH: 'tests/e2e/results.json' + WCPAY_USE_BUILD_ARTIFACT: ${{ inputs.wcpay-use-build-artifact }} + WCPAY_ARTIFACT_DIRECTORY: 'zipfile' + NODE_ENV: 'test' + FORCE_E2E_DEPS_SETUP: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + wcpay-e2e-tests-only-changed: + runs-on: ubuntu-latest + if: github.event.pull_request.draft == true + name: WC - latest + + env: + E2E_WP_VERSION: 'latest' + E2E_WC_VERSION: 'latest' + + steps: + - name: Checkout WCPay repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.repo-branch || github.ref }} + fetch-depth: 0 + + - name: 'Download WooCommerce Payments build file' + if: ${{ inputs.wcpay-use-build-artifact }} + uses: actions/download-artifact@v4 + with: + name: woocommerce-payments + path: ${{ env.WCPAY_ARTIFACT_DIRECTORY }} + + - name: Setup E2E environment + uses: ./.github/actions/e2e/env-setup + + - name: Install Playwright + shell: bash + run: npx playwright install chromium + + - name: Run tests, upload screenshots & logs + uses: ./.github/actions/e2e-pw/run-changed-tests diff --git a/.github/workflows/e2e-pw-pull-request.yml b/.github/workflows/e2e-pw-pull-request.yml index da6765fb51b..38d496b4def 100644 --- a/.github/workflows/e2e-pw-pull-request.yml +++ b/.github/workflows/e2e-pw-pull-request.yml @@ -6,6 +6,9 @@ on: - develop - trunk workflow_dispatch: + pull_request_review: + types: + - submitted workflow_call: inputs: wcpay-use-build-artifact: @@ -43,7 +46,7 @@ concurrency: jobs: wcpay-e2e-tests: runs-on: ubuntu-latest - + if: github.event.pull_request.draft == false name: WC - latest env: diff --git a/changelog/dev-add-workflow-to-run-only-changed-e2e-tests b/changelog/dev-add-workflow-to-run-only-changed-e2e-tests new file mode 100644 index 00000000000..8339a98a522 --- /dev/null +++ b/changelog/dev-add-workflow-to-run-only-changed-e2e-tests @@ -0,0 +1,4 @@ +Significance: patch +Type: dev + +Add workflow that only runs changed E2E tests, and succeeds if nothing changed diff --git a/package-lock.json b/package-lock.json index 488b8164029..c603438e519 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,7 @@ "devDependencies": { "@automattic/color-studio": "2.3.1", "@jest/test-sequencer": "29.5.0", - "@playwright/test": "1.43.1", + "@playwright/test": "1.50.0", "@pmmmwh/react-refresh-webpack-plugin": "0.5.7", "@testing-library/jest-dom": "5.14.1", "@testing-library/react": "11.2.5", @@ -1878,6 +1878,26 @@ } } }, + "node_modules/@automattic/puppeteer-utils/node_modules/fsevents": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", + "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "deprecated": "Upgrade to fsevents v2 to mitigate potential security issues", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "dependencies": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + }, + "engines": { + "node": ">= 4.0" + } + }, "node_modules/@automattic/puppeteer-utils/node_modules/get-stream": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", @@ -8409,18 +8429,19 @@ } }, "node_modules/@playwright/test": { - "version": "1.43.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.43.1.tgz", - "integrity": "sha512-HgtQzFgNEEo4TE22K/X7sYTYNqEMMTZmFS8kTq6m8hXj+m1D8TgwgIbumHddJa9h4yl4GkKb8/bgAl2+g7eDgA==", + "version": "1.50.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.50.0.tgz", + "integrity": "sha512-ZGNXbt+d65EGjBORQHuYKj+XhCewlwpnSd/EDuLPZGSiEWmgOJB5RmMCCYGy5aMfTs9wx61RivfDKi8H/hcMvw==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "playwright": "1.43.1" + "playwright": "1.50.0" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=16" + "node": ">=18" } }, "node_modules/@pmmmwh/react-refresh-webpack-plugin": { @@ -22984,6 +23005,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -28573,6 +28605,14 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "license": "MIT", + "optional": true + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -29056,6 +29096,21 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -38454,6 +38509,14 @@ "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", "dev": true }, + "node_modules/nan": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", + "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", + "dev": true, + "license": "MIT", + "optional": true + }, "node_modules/nanoid": { "version": "3.3.7", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", @@ -39804,33 +39867,35 @@ } }, "node_modules/playwright": { - "version": "1.43.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.43.1.tgz", - "integrity": "sha512-V7SoH0ai2kNt1Md9E3Gwas5B9m8KR2GVvwZnAI6Pg0m3sh7UvgiYhRrhsziCmqMJNouPckiOhk8T+9bSAK0VIA==", + "version": "1.50.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.50.0.tgz", + "integrity": "sha512-+GinGfGTrd2IfX1TA4N2gNmeIksSb+IAe589ZH+FlmpV3MYTx6+buChGIuDLQwrGNCw2lWibqV50fU510N7S+w==", "dev": true, + "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.43.1" + "playwright-core": "1.50.0" }, "bin": { "playwright": "cli.js" }, "engines": { - "node": ">=16" + "node": ">=18" }, "optionalDependencies": { "fsevents": "2.3.2" } }, "node_modules/playwright-core": { - "version": "1.43.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.43.1.tgz", - "integrity": "sha512-EI36Mto2Vrx6VF7rm708qSnesVQKbxEWvPrfA1IPY6HgczBplDx7ENtx+K2n4kJ41sLLkuGfmb0ZLSSXlDhqPg==", + "version": "1.50.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.50.0.tgz", + "integrity": "sha512-CXkSSlr4JaZs2tZHI40DsZUN/NIwgaUPsyLuOAaIZp2CyF2sN5MM5NJsyB188lFSSozFxQ5fPT4qM+f0tH/6wQ==", "dev": true, + "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" }, "engines": { - "node": ">=16" + "node": ">=18" } }, "node_modules/please-upgrade-node": { diff --git a/package.json b/package.json index 20ab8230229..b73ba28825a 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "test:e2e-pw-update-snapshots": "npm run test:e2e-pw -- --update-snapshots", "test:e2e-pw-ui": "./tests/e2e-pw/test-e2e-pw-ui.sh", "test:e2e-pw-ci": "npx playwright test --config=tests/e2e-pw/playwright.config.ts --grep-invert @todo", + "test:e2e-pw-ci-only-changed": "npx playwright test --only-changed=origin/develop --pass-with-no-tests --config=tests/e2e-pw/playwright.config.ts --grep-invert @todo", "test:update-snapshots": "npm run test:js -- --updateSnapshot", "test:php": "./bin/run-tests.sh", "test:php-coverage": "./bin/check-test-coverage.sh", @@ -96,7 +97,7 @@ "devDependencies": { "@automattic/color-studio": "2.3.1", "@jest/test-sequencer": "29.5.0", - "@playwright/test": "1.43.1", + "@playwright/test": "1.50.0", "@pmmmwh/react-refresh-webpack-plugin": "0.5.7", "@testing-library/jest-dom": "5.14.1", "@testing-library/react": "11.2.5",