From 067e4fa304ef16b6efcc0ff2ff7b64c6ce65a01d Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 17 Apr 2023 14:24:23 +0000 Subject: [PATCH] test: fix flaky tests (#26505) * test(launchpad): fix flaky tests Closes #23153 Closes #23154 --------- Co-authored-by: astone123 --- .circleci/workflows.yml | 2 +- packages/app/cypress/e2e/runs.cy.ts | 2 +- .../e2e/config-files-error-handling.cy.ts | 46 +++++++++---------- .../cypress/e2e/config-warning.cy.ts | 11 ++--- .../launchpad/cypress/e2e/migration.cy.ts | 4 +- .../launchpad/cypress/e2e/project-setup.cy.ts | 15 +++--- .../cypress/e2e/scaffold-project.cy.ts | 10 +++- .../cypress/e2e/top-nav-launchpad.cy.ts | 8 ++-- 8 files changed, 52 insertions(+), 46 deletions(-) diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml index d3a1c8fa422d..5566f6741db9 100644 --- a/.circleci/workflows.yml +++ b/.circleci/workflows.yml @@ -69,7 +69,7 @@ windowsWorkflowFilters: &windows-workflow-filters or: - equal: [ develop, << pipeline.git.branch >> ] # use the following branch as well to ensure that v8 snapshot cache updates are fully tested - - equal: [ 'lmiller/fixing-vite-windows', << pipeline.git.branch >> ] + - equal: [ 'windows-flake', << pipeline.git.branch >> ] - equal: [ 'update-v8-snapshot-cache-on-develop', << pipeline.git.branch >> ] - matches: pattern: /^release\/\d+\.\d+\.\d+$/ diff --git a/packages/app/cypress/e2e/runs.cy.ts b/packages/app/cypress/e2e/runs.cy.ts index e64a589d973a..f9df449949c4 100644 --- a/packages/app/cypress/e2e/runs.cy.ts +++ b/packages/app/cypress/e2e/runs.cy.ts @@ -323,7 +323,7 @@ describe('App: Runs', { viewportWidth: 1200 }, () => { moveToRunsPage() cy.findByText(defaultMessages.runs.connect.buttonProject).click() cy.contains('button', defaultMessages.runs.connect.modal.selectProject.createProject).click() - cy.findByText(defaultMessages.runs.connectSuccessAlert.title).should('be.visible') + cy.findByText(defaultMessages.runs.connectSuccessAlert.title, { timeout: 10000 }).should('be.visible') cy.withCtx(async (ctx) => { const config = await ctx.project.getConfig() diff --git a/packages/launchpad/cypress/e2e/config-files-error-handling.cy.ts b/packages/launchpad/cypress/e2e/config-files-error-handling.cy.ts index 89c8d832ed72..9b58c8385119 100644 --- a/packages/launchpad/cypress/e2e/config-files-error-handling.cy.ts +++ b/packages/launchpad/cypress/e2e/config-files-error-handling.cy.ts @@ -31,7 +31,7 @@ describe('Config files error handling', () => { cy.findByRole('button', { name: 'Try again' }).click() - cy.get('h1').should('contain', 'Welcome to Cypress') + cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 }) }) it('shows the upgrade screen if there is a legacy config file', () => { @@ -70,7 +70,7 @@ describe('Config files error handling', () => { cy.findByRole('button', { name: 'Try again' }).click() - cy.get('h1').should('contain', 'Welcome to Cypress') + cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 }) }) it('handles deprecated config fields', () => { @@ -92,7 +92,7 @@ describe('Config files error handling', () => { cy.findByRole('button', { name: 'Try again' }).click() - cy.get('h1').should('contain', 'Choose a browser') + cy.contains('h1', 'Choose a browser', { timeout: 10000 }) }) }) @@ -102,7 +102,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('plugins-root-sync-error', ['--e2e']) cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) expectStackToBe('open') }) @@ -111,7 +111,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('plugins-root-syntax-error', ['--e2e']) cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) expectStackToBe('open') }) @@ -120,7 +120,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('plugins-root-async-error', ['--e2e']) cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', 'Unexpected Error') + cy.contains('h1', 'Unexpected Error', { timeout: 10000 }) expectStackToBe('open') }) @@ -129,7 +129,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('plugins-function-sync-error', ['--e2e']) cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) expectStackToBe('open') }) @@ -138,7 +138,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('config-with-invalid-browser', ['--e2e']) cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) expectStackToBe('closed') }) @@ -147,7 +147,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('plugins-function-sync-error', ['--e2e']) cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) expectStackToBe('open') }) @@ -156,7 +156,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('config-with-ts-syntax-error') cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.percySnapshot() cy.withCtx(async (ctx) => { @@ -165,7 +165,7 @@ describe('Launchpad: Error System Tests', () => { cy.findByRole('button', { name: 'Try again' }).click() - cy.get('h1').should('contain', 'Welcome to Cypress') + cy.contains('h1', 'Welcome to Cypress', { timeout: 10000 }) }) it(`clears the error correctly after first 'try again' attempt`, () => { @@ -174,7 +174,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('config-with-ts-syntax-error') cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) // Try again while the config is still invalid cy.findByRole('button', { name: 'Try again' }).click() @@ -182,7 +182,7 @@ describe('Launchpad: Error System Tests', () => { cy.wait('@resetErrorsAndLoadConfig') // Wait until config error is on screen again - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.withCtx(async (ctx) => { await ctx.actions.file.writeFileInProject('cypress.config.ts', 'export default { e2e: { supportFile: false } }') @@ -190,7 +190,7 @@ describe('Launchpad: Error System Tests', () => { cy.findByRole('button', { name: 'Try again' }).click() - cy.get('h1').should('contain', 'Welcome to Cypress') + cy.contains('h1', 'Welcome to Cypress') }) it('shows correct user file instead of node file', () => { @@ -198,7 +198,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('config-with-import-error') cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.percySnapshot() cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.js:3:23') @@ -209,7 +209,7 @@ describe('Launchpad: Error System Tests', () => { cy.openProject('config-with-ts-module-error') cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.percySnapshot() cy.get('[data-testid="error-code-frame"]').should('contain', 'cypress.config.ts:6:10') @@ -223,7 +223,7 @@ describe('setupNodeEvents', () => { cy.visitLaunchpad() cy.skipWelcome() cy.findByText('E2E Testing').click() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.percySnapshot() }) @@ -233,7 +233,7 @@ describe('setupNodeEvents', () => { cy.visitLaunchpad() cy.skipWelcome() cy.findByText('E2E Testing').click() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.percySnapshot() cy.get('[data-cy="alert-body"]').should('contain', 'integrationFolder') @@ -245,7 +245,7 @@ describe('setupNodeEvents', () => { cy.visitLaunchpad() cy.skipWelcome() cy.findByText('E2E Testing').click() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.percySnapshot() }) @@ -279,7 +279,7 @@ describe('setupNodeEvents', () => { cy.findByRole('button', { name: 'Try again' }).click() - cy.get('h1').should('contain', 'Choose a browser') + cy.contains('h1', 'Choose a browser', { timeout: 10000 }) }) it('handles multiple config errors and then recovers', () => { @@ -293,7 +293,7 @@ describe('setupNodeEvents', () => { cy.visitLaunchpad() cy.skipWelcome() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.get('[data-cy="alert-body"]').should('contain', 'Expected baseUrl to be a fully qualified URL') cy.withCtx(async (ctx) => { @@ -302,7 +302,7 @@ describe('setupNodeEvents', () => { cy.findByRole('button', { name: 'Try again' }).click() cy.get('[data-cy-testingType=e2e]').click() - cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle) + cy.contains('h1', cy.i18n.launchpadErrors.generic.configErrorTitle, { timeout: 10000 }) cy.get('[data-cy="alert-body"]').should('contain', 'The baseUrl configuration option is now invalid when set from the root of the config object') cy.withCtx(async (ctx) => { @@ -310,7 +310,7 @@ describe('setupNodeEvents', () => { }) cy.findByRole('button', { name: 'Try again' }).click() - cy.get('h1').should('contain', 'Choose a browser') + cy.contains('h1', 'Choose a browser', { timeout: 10000 }) cy.get('[data-cy="alert"]').should('contain', 'Warning: Cannot Connect Base Url Warning') }) diff --git a/packages/launchpad/cypress/e2e/config-warning.cy.ts b/packages/launchpad/cypress/e2e/config-warning.cy.ts index 66d925c14bcf..46e3b05df14e 100644 --- a/packages/launchpad/cypress/e2e/config-warning.cy.ts +++ b/packages/launchpad/cypress/e2e/config-warning.cy.ts @@ -232,7 +232,7 @@ describe('component testing dependency warnings', () => { cy.skipWelcome() cy.contains('component-testing-outdated-dependencies').click() cy.get('[data-cy="warning-alert"]').should('not.exist') - cy.get('a').contains('Projects').click() + cy.contains('a', 'Projects').click() cy.get('[data-cy-testingtype="component"]').click() cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist') .should('contain.text', 'Warning: Component Testing Mismatched Dependencies') @@ -243,8 +243,7 @@ describe('component testing dependency warnings', () => { cy.get('.warning-markdown').find('li').should('have.length', 3) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23154 - it('warns against outdated @vue/cli dependency', { retries: 15 }, () => { + it('warns against outdated @vue/cli dependency', () => { cy.scaffoldProject('outdated-deps-vuecli3') cy.addProject('outdated-deps-vuecli3') cy.openGlobalMode() @@ -252,8 +251,8 @@ describe('component testing dependency warnings', () => { cy.skipWelcome() cy.contains('outdated-deps-vuecli3').click() cy.get('[data-cy="warning-alert"]').should('not.exist') - cy.get('a').contains('Projects').click() - cy.get('[data-cy-testingtype="component"]').click() + cy.contains('a', 'Projects').click() + cy.get('[data-cy-testingtype="component"]', { timeout: 12000 }).click() cy.get('[data-cy="warning-alert"]', { timeout: 12000 }).should('exist') .should('contain.text', 'Warning: Component Testing Mismatched Dependencies') .should('contain.text', '@vue/cli-service. Expected ^=4.0.0 || ^=5.0.0, found 3.12.1.') @@ -271,7 +270,7 @@ describe('component testing dependency warnings', () => { cy.contains('vueclivue3-configured').click() cy.get('[data-cy="warning-alert"]').should('not.exist') cy.get('a').contains('Projects').click() - cy.get('[data-cy-testingtype="component"]').click() + cy.get('[data-cy-testingtype="component"]', { timeout: 10000 }).click() // Wait until launch browser screen and assert warning does not exist cy.contains('Choose a browser', { timeout: 12000 }) diff --git a/packages/launchpad/cypress/e2e/migration.cy.ts b/packages/launchpad/cypress/e2e/migration.cy.ts index 10d891eb24be..7b1e21ea05c4 100644 --- a/packages/launchpad/cypress/e2e/migration.cy.ts +++ b/packages/launchpad/cypress/e2e/migration.cy.ts @@ -59,7 +59,7 @@ function finishMigrationAndContinue () { } function checkOutcome () { - cy.contains('Welcome to Cypress!').should('be.visible') + cy.contains('Welcome to Cypress!', { timeout: 10000 }).should('be.visible') } function runAutoRename () { @@ -95,7 +95,7 @@ describe('global mode', () => { cy.contains('migration-e2e-export-default').click() // rename integration->e2e - cy.get(renameAutoStep).should('exist') + cy.get(renameAutoStep, { timeout: 10000 }).should('exist') cy.get(renameManualStep).should('not.exist') // cypress/support/index.ts -> cypress/support/e2e.ts diff --git a/packages/launchpad/cypress/e2e/project-setup.cy.ts b/packages/launchpad/cypress/e2e/project-setup.cy.ts index 478938ab1051..ec491695553e 100644 --- a/packages/launchpad/cypress/e2e/project-setup.cy.ts +++ b/packages/launchpad/cypress/e2e/project-setup.cy.ts @@ -556,8 +556,7 @@ describe('Launchpad: Setup Project', () => { }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153 - describe('Command for package managers', { retries: 15 }, () => { + describe('Command for package managers', () => { it('makes the right command for yarn', () => { scaffoldAndOpenProject('pristine-yarn') @@ -566,7 +565,7 @@ describe('Launchpad: Setup Project', () => { cy.get('[data-cy-testingtype="component"]').click() cy.get('[data-testid="select-framework"]').click() cy.findByText('Create React App').click() - cy.findByText('Next step').click() + cy.contains('button', 'Next step').should('not.be.disabled').click() cy.findByDisplayValue('yarn add -D react-scripts react-dom react').should('be.visible') }) @@ -578,7 +577,7 @@ describe('Launchpad: Setup Project', () => { cy.get('[data-cy-testingtype="component"]').click() cy.get('[data-testid="select-framework"]').click() cy.findByText('Create React App').click() - cy.findByText('Next step').click() + cy.contains('button', 'Next step').should('not.be.disabled').click() cy.findByDisplayValue('pnpm install -D react-scripts react-dom react') }) @@ -588,9 +587,9 @@ describe('Launchpad: Setup Project', () => { cy.visitLaunchpad() cy.get('[data-cy-testingtype="component"]').click() - cy.get('button').should('be.visible').contains('Vue.js 3(detected)') - cy.get('button').should('be.visible').contains('Vite(detected)') - cy.findByText('Next step').click() + cy.contains('button', 'Vue.js 3(detected)').should('be.visible') + cy.contains('button', 'Vite(detected)').should('be.visible') + cy.contains('button', 'Next step').should('not.be.disabled').click() cy.findByTestId('alert').contains(`You've successfully installed all required dependencies.`) }) @@ -602,7 +601,7 @@ describe('Launchpad: Setup Project', () => { cy.get('[data-cy-testingtype="component"]').click() cy.get('[data-testid="select-framework"]').click() cy.findByText('Create React App').click() - cy.findByText('Next step').click() + cy.contains('button', 'Next step').should('not.be.disabled').click() cy.findByDisplayValue('npm install -D react-scripts react-dom react') }) }) diff --git a/packages/launchpad/cypress/e2e/scaffold-project.cy.ts b/packages/launchpad/cypress/e2e/scaffold-project.cy.ts index 855f25ed4b81..3af6d5351908 100644 --- a/packages/launchpad/cypress/e2e/scaffold-project.cy.ts +++ b/packages/launchpad/cypress/e2e/scaffold-project.cy.ts @@ -168,6 +168,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => { }) it('generates valid config file for pristine project without cypress installed', () => { + cy.intercept('mutation-ScaffoldedFiles_completeSetup').as('mutationScaffoldedFiles') + cy.intercept('query-MainLaunchpadQuery').as('mainLaunchpadQuery') + cy.intercept('query-HeaderBar_HeaderBarQuery').as('headerBarQuery') + cy.intercept('query-CloudViewerAndProject_RequiredData').as('cloudViewerAndProjectRequiredData') cy.scaffoldProject('pristine') cy.openProject('pristine') cy.withCtx((ctx) => ctx.currentProject).then((currentProject) => { @@ -178,6 +182,10 @@ describe('scaffolding new projects', { defaultCommandTimeout: 7000 }, () => { cy.skipWelcome() cy.contains('button', cy.i18n.testingType.e2e.name).click() cy.contains('button', cy.i18n.setupPage.step.continue).click() - cy.contains('h1', cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible') + cy.wait('@mutationScaffoldedFiles') + cy.wait('@mainLaunchpadQuery') + cy.wait('@headerBarQuery') + cy.wait('@cloudViewerAndProjectRequiredData') + cy.get('h1').contains(cy.i18n.setupPage.testingCard.chooseABrowser).should('be.visible') }) }) diff --git a/packages/launchpad/cypress/e2e/top-nav-launchpad.cy.ts b/packages/launchpad/cypress/e2e/top-nav-launchpad.cy.ts index efac8bb1ffa3..626a2ff7e58e 100644 --- a/packages/launchpad/cypress/e2e/top-nav-launchpad.cy.ts +++ b/packages/launchpad/cypress/e2e/top-nav-launchpad.cy.ts @@ -366,12 +366,12 @@ describe('Launchpad Top Nav Workflows', () => { options.sinon.stub(ctx._apis.authApi, 'logIn').callsFake(async (onMessage) => { setTimeout(() => { onMessage({ browserOpened: true }) - }, 500) + }, 2000) return new Promise((resolve) => { setTimeout(() => { resolve(options.user) - }, 2000) + }, 3000) }) }) }, { user }) @@ -687,7 +687,7 @@ describe('Launchpad Top Nav Workflows', () => { cy.get('[data-cy="project-card"]').click() - cy.contains('E2E Testing').click() + cy.contains('E2E Testing', { timeout: 10000 }).click() mockLogInActionsForUser(mockUser) logIn({ expectedNextStepText: 'Continue', displayName: mockUser.name }) @@ -707,7 +707,7 @@ describe('Launchpad Top Nav Workflows', () => { cy.get('[data-cy="project-card"]').click() - cy.contains('E2E Testing').click() + cy.contains('E2E Testing', { timeout: 10000 }).click() mockLogInActionsForUser(mockUser) logIn({ expectedNextStepText: 'Connect project', displayName: mockUser.name })