diff --git a/e2e/qwik-cli-e2e/README.md b/e2e/qwik-cli-e2e/README.md new file mode 100644 index 00000000000..959a75de13a --- /dev/null +++ b/e2e/qwik-cli-e2e/README.md @@ -0,0 +1,51 @@ +# CLI E2E tests for the Qwik Framework + +This package provides isolated E2E tests by generating a new application with local packages and then running tests again that code by executing it and verifying expected behavior. + +## Description + +Tests can be invoked by running `pnpm run test.e2e-cli`. + +**Note that running E2E tests requires the workspace projects to be prebuilt manually!** + +E2E project does the following internally: + +0. Vitest is configured to run a setup function once **PRIOR TO ALL** tests. During the setup `@builder.io/qwik`, `@builder.io/qwik-city` and `eslint-plugin-qwik` packages will be packed with `pnpm pack` Those will be used at a step 2 for every test. Tarballs are located in `temp/tarballs` folder within this repo. It is assumed that packages are built before E2E is executed. + +1. Simulates `npm create qwik` locally using direct command `node packages/create-qwik/create-qwik.cjs playground {outputDir}` + + - By default `outputDir` is an auto-generated one using `tmp` npm package. The application that is created here will be removed after the test is executed + - It is possible to install into custom folder using environment variable `TEMP_E2E_PATH`. Here's how the command would look like in this case: + - with absolute path `TEMP_E2E_PATH=/Users/name/projects/tests pnpm run test.e2e-cli` + - with path relative to the qwik workspace `TEMP_E2E_PATH=temp/e2e-folder pnpm run test.e2e-cli` + + Note that provided folder should exist. If custom path is used, generated application will not be removed after the test completes, which is helpful for debugging. + +2. Uses packed `@builder.io/qwik`, `@builder.io/qwik-city` and `eslint-plugin-qwik` packages to update package.json file of the generated application with `file:path-to-package.tgz`. + +3. Runs actual tests. Please pay attention at the `beforeAll` hook in the spec file + +```typescript +beforeAll(() => { + const config = scaffoldQwikProject(); + global.tmpDir = config.tmpDir; + + return async () => { + await killAllRegisteredProcesses(); + config.cleanupFn(); + }; +}); +``` + +Notice that `beforeAll` returns a function, which will be executed after the test completes either with success or failure. + +Both `config.cleanupFn();` and `killAllRegisteredProcesses` there are extremely important: + +- `config.cleanupFn()` is used in order to remove temporary folder with generated project after the test is executed (again, it's not being removed if `TEMP_E2E_PATH` is provided). +- `killAllRegisteredProcesses` should be used to remove any active ports as we are serving the app during the test execution. + Processes are being registered internally when `runCommandUntil` is executed. If you're executing something manually, you can use `registerExecutedChildProcess` utility to register the process. + Despite `killAllRegisteredProcesses` will kill all processes when test exists, it is also a good practice to kill the process manually within the `it` statement using `await promisifiedTreeKill(yourChildProcess.pId, 'SIGKILL')` + +## Adding new tests + +Right now we have only one test file within this project. This means only one test application will be created and used, which is good from the execution time standpoint. If more files are added, it shouldn't potentially be a problem as we have `fileParallelism: false` set in the `vite.config.ts`, which means only one test will be executed at a time. This obviously slows down the execution time, but is safer, because we're working with a real file system. diff --git a/e2e/qwik-cli-e2e/package.json b/e2e/qwik-cli-e2e/package.json new file mode 100644 index 00000000000..71fe36bbce5 --- /dev/null +++ b/e2e/qwik-cli-e2e/package.json @@ -0,0 +1,11 @@ +{ + "name": "qwik-cli-e2e", + "private": true, + "scripts": { + "e2e": "vitest run --config=vite.config.ts", + "e2e:watch": "vitest watch --config=vite.config.ts" + }, + "dependencies": { + "kleur": "4.1.5" + } +} diff --git a/e2e/qwik-cli-e2e/tests/serve.spec.ts b/e2e/qwik-cli-e2e/tests/serve.spec.ts new file mode 100644 index 00000000000..c30d8aa30a9 --- /dev/null +++ b/e2e/qwik-cli-e2e/tests/serve.spec.ts @@ -0,0 +1,64 @@ +import { assert, test, beforeAll, expect } from 'vitest'; +import { + assertHostUnused, + getPageHtml, + promisifiedTreeKill, + killAllRegisteredProcesses, + runCommandUntil, + scaffoldQwikProject, + DEFAULT_TIMEOUT, +} from '../utils'; +import { existsSync, readFileSync, writeFileSync } from 'fs'; +import { join } from 'path'; + +const SERVE_PORT = 3535; +beforeAll(() => { + const config = scaffoldQwikProject(); + global.tmpDir = config.tmpDir; + + return async () => { + await killAllRegisteredProcesses(); + config.cleanupFn(); + }; +}); + +test( + 'Should serve the app in dev mode and update the content on hot reload', + { timeout: DEFAULT_TIMEOUT }, + async () => { + const host = `http://localhost:${SERVE_PORT}/`; + await assertHostUnused(host); + const p = await runCommandUntil( + `npm run dev -- --port ${SERVE_PORT}`, + global.tmpDir, + (output) => { + return output.includes(host); + } + ); + assert.equal(existsSync(global.tmpDir), true); + + await expectHtmlOnARootPage(host); + + await promisifiedTreeKill(p.pid!, 'SIGKILL'); + } +); + +async function expectHtmlOnARootPage(host: string) { + expect((await getPageHtml(host)).querySelector('.container h1')?.textContent).toBe( + `So fantasticto have you here` + ); + const heroComponentPath = join(global.tmpDir, `src/components/starter/hero/hero.tsx`); + const heroComponentTextContent = readFileSync(heroComponentPath, 'utf-8'); + writeFileSync( + heroComponentPath, + heroComponentTextContent.replace( + `to have you here`, + `to have e2e tests here` + ) + ); + // wait for the arbitrary amount of time before the app is reloaded + await new Promise((r) => setTimeout(r, 2000)); + expect((await getPageHtml(host)).querySelector('.container h1')?.textContent).toBe( + `So fantasticto have e2e tests here` + ); +} diff --git a/e2e/qwik-cli-e2e/tsconfig.json b/e2e/qwik-cli-e2e/tsconfig.json new file mode 100644 index 00000000000..8ba4d83acea --- /dev/null +++ b/e2e/qwik-cli-e2e/tsconfig.json @@ -0,0 +1,7 @@ +{ + "compilerOptions": { + "types": ["vitest/globals"], + "esModuleInterop": true + }, + "include": ["utils", "tests"] +} diff --git a/e2e/qwik-cli-e2e/utils/index.ts b/e2e/qwik-cli-e2e/utils/index.ts new file mode 100644 index 00000000000..c67a7636897 --- /dev/null +++ b/e2e/qwik-cli-e2e/utils/index.ts @@ -0,0 +1,197 @@ +import { ChildProcess, exec, execSync } from 'child_process'; +import { dirSync } from 'tmp'; +import { existsSync, mkdirSync, readFileSync, rmSync, rmdirSync, writeFileSync } from 'fs'; +import { promisify } from 'util'; +import { resolve, join, dirname } from 'path'; +import treeKill from 'tree-kill'; +import { yellow } from 'kleur/colors'; +import { createDocument } from '../../../packages/qwik/src/testing/document'; + +export function scaffoldQwikProject(): { tmpDir: string; cleanupFn: () => void } { + const tmpHostDirData = getTmpDirSync(process.env.TEMP_E2E_PATH); + const cleanupFn = () => { + if (!tmpHostDirData.overridden) { + cleanup(tmpHostDirData.path); + } else { + log('Custom E2E test path was used, skipping the removal of test folder'); + } + }; + try { + const tmpDir = runCreateQwikCommand(tmpHostDirData.path); + log(`Created test application at "${tmpDir}"`); + replacePackagesWithLocalOnes(tmpDir); + return { cleanupFn, tmpDir }; + } catch (error) { + cleanupFn(); + throw error; + } +} + +function cleanup(tmpDir: string) { + log(`Removing tmp dir "${tmpDir}"`); + rmSync(tmpDir, { recursive: true }); +} + +function getTmpDirSync(tmpDirOverride?: string) { + if (tmpDirOverride) { + tmpDirOverride = resolve(workspaceRoot, tmpDirOverride); + } + + if (tmpDirOverride && !existsSync(tmpDirOverride)) { + throw new Error(`"${tmpDirOverride}" does not exist.`); + } + if (tmpDirOverride) { + const p = join(tmpDirOverride, 'qwik_e2e'); + if (existsSync(p)) { + log(`Removing project folder "${p}" (will be recreated).`); + rmSync(p, { recursive: true }); + } + mkdirSync(p); + return { path: p, overridden: true }; + } + return { path: dirSync({ prefix: 'qwik_e2e' }).name, overridden: false }; +} + +function runCreateQwikCommand(tmpDir: string): string { + const appDir = 'e2e-app'; + execSync( + `node "${workspaceRoot}/packages/create-qwik/create-qwik.cjs" playground "${join(tmpDir, appDir)}"` + ); + return join(tmpDir, appDir); +} + +function replacePackagesWithLocalOnes(tmpDir: string) { + const tarballConfig = JSON.parse( + readFileSync(join(workspaceRoot, 'temp/tarballs/paths.json'), 'utf-8') + ); + for (const { name, absolutePath } of tarballConfig) { + patchPackageJsonForPlugin(tmpDir, name, absolutePath); + } + execSync('npm i', { + cwd: tmpDir, + // only output errors + stdio: ['ignore', 'ignore', 'inherit'], + }); +} + +function patchPackageJsonForPlugin(tmpDirName: string, npmPackageName: string, distPath: string) { + const path = join(tmpDirName, 'package.json'); + const json = JSON.parse(readFileSync(path, 'utf-8')); + json.devDependencies[npmPackageName] = `file:${distPath}`; + writeFileSync(path, JSON.stringify(json)); +} + +export function registerExecutedChildProcess(process: ChildProcess) { + if (typeof global !== 'undefined') { + (global.pIds ??= []).push(process.pid); + log(`Registered a process with id "${process.pid}"`); + } else { + throw new Error('"global" is not defined'); + } +} + +export function runCommandUntil( + command: string, + tmpDir: string, + criteria: (output: string) => boolean +): Promise { + const p = exec(command, { + cwd: tmpDir, + encoding: 'utf-8', + }); + registerExecutedChildProcess(p); + return new Promise((res, rej) => { + let output = ''; + let complete = false; + + function checkCriteria(c: any) { + output += c.toString(); + if (criteria(stripConsoleColors(output)) && !complete) { + complete = true; + res(p); + } + } + + p.stdout?.on('data', checkCriteria); + p.stderr?.on('data', checkCriteria); + p.on('exit', (code) => { + if (!complete) { + rej(`Exited with ${code}`); + } else { + res(p); + } + }); + }); +} + +function stripConsoleColors(log: string): string { + return log.replace( + // eslint-disable-next-line no-control-regex + /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, + '' + ); +} + +export async function getPageHtml(pageUrl: string): Promise { + const res = await fetch(pageUrl, { headers: { accept: 'text/html' } }).then((r) => r.text()); + return createDocument({ html: res }); +} + +export async function assertHostUnused(host: string): Promise { + try { + const response = await fetch(host, { headers: { accept: 'text/html' } }); + } catch (error) { + // TODO: test this in different environments + if (error.cause.code === 'ECONNREFUSED') { + return; + } + } + throw new Error(`Host ${host} is already in use!`); +} + +// promisify fails to get the proper type overload, so manually enforcing the type +const _promisifiedTreeKill = promisify(treeKill) as (pid: number, signal: string) => Promise; + +export const promisifiedTreeKill = (pid: number, signal: string) => { + try { + return _promisifiedTreeKill(pid, signal); + } catch (error) { + console.error('Failed to kill the process ' + pid, error); + } +}; + +export async function killAllRegisteredProcesses() { + const pIds = (global?.pIds as number[]) ?? []; + const result = await Promise.allSettled(pIds.map((pId) => promisifiedTreeKill(pId, 'SIGKILL'))); + const stringifiedResult = JSON.stringify( + result.map((v, i) => ({ + pId: pIds[i], + status: v.status === 'fulfilled' ? 'success' : 'failure', + })) + ); + log('Cleaned up processes invoked by e2e test: ' + stringifiedResult); +} + +export const workspaceRoot = _computeWorkspaceRoot(process.cwd()); + +function _computeWorkspaceRoot(cwd: string) { + if (dirname(cwd) === cwd) { + return process.cwd(); + } + + const packageJsonAtCwd = join(cwd, 'package.json'); + if (existsSync(packageJsonAtCwd)) { + const content = JSON.parse(readFileSync(packageJsonAtCwd, 'utf-8')); + if (content.name === 'qwik-monorepo') { + return cwd; + } + } + return _computeWorkspaceRoot(dirname(cwd)); +} + +export function log(text: string) { + // eslint-disable-next-line no-console + console.log(yellow('E2E: ' + text)); +} + +export const DEFAULT_TIMEOUT = 30000; diff --git a/e2e/qwik-cli-e2e/utils/setup.ts b/e2e/qwik-cli-e2e/utils/setup.ts new file mode 100644 index 00000000000..60096c0b3ad --- /dev/null +++ b/e2e/qwik-cli-e2e/utils/setup.ts @@ -0,0 +1,41 @@ +import { execSync } from 'child_process'; +import { join } from 'path'; +import { workspaceRoot } from '.'; +import { existsSync, writeFileSync } from 'fs'; + +const packageCfg = { + '@builder.io/qwik': { + packagePath: 'packages/qwik', + distPath: 'packages/qwik/dist', + }, + '@builder.io/qwik-city': { + packagePath: 'packages/qwik-city', + distPath: 'packages/qwik-city/lib', + }, + 'eslint-plugin-qwik': { + packagePath: 'packages/eslint-plugin-qwik', + distPath: 'packages/eslint-plugin-qwik/dist', + }, +}; +function ensurePackageBuilt() { + for (const [name, cfg] of Object.entries(packageCfg)) { + if (!existsSync(join(workspaceRoot, cfg.distPath))) { + throw new Error(`Looks like package "${name}" has not been built yet.`); + } + } +} +function packPackages() { + const tarballPaths: { name: string; absolutePath: string }[] = []; + const tarballOutDir = join(workspaceRoot, 'temp', 'tarballs'); + for (const [name, cfg] of Object.entries(packageCfg)) { + const out = execSync(`pnpm pack --pack-destination=${tarballOutDir}`, { + cwd: join(workspaceRoot, cfg.packagePath), + encoding: 'utf-8', + }); + tarballPaths.push({ name, absolutePath: out.replace(/(\r\n|\n|\r)/gm, '') }); + } + writeFileSync(join(tarballOutDir, 'paths.json'), JSON.stringify(tarballPaths)); +} + +ensurePackageBuilt(); +packPackages(); diff --git a/e2e/qwik-cli-e2e/vite.config.ts b/e2e/qwik-cli-e2e/vite.config.ts new file mode 100644 index 00000000000..a61c832be81 --- /dev/null +++ b/e2e/qwik-cli-e2e/vite.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vitest/config'; +import tsconfigPaths from 'vite-tsconfig-paths'; + +export default defineConfig({ + plugins: [tsconfigPaths({ ignoreConfigErrors: true, root: '../../' })], + test: { + include: ['./tests/*.spec.?(c|m)[jt]s?(x)'], + setupFiles: ['./utils/setup.ts'], + // Run only one test at a time to avoid potential conflicts. + // These tests interact directly with the filesystem and/or run processes on localhost, + // which can lead to issues if multiple tests are executed simultaneously + fileParallelism: false, + }, +}); diff --git a/package.json b/package.json index fb47d2b93fd..bdba878c821 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "@types/prompts": "2.4.9", "@types/react": "18.3.3", "@types/semver": "7.5.8", + "@types/tmp": "0.2.6", "@types/which-pm-runs": "1.0.2", "@typescript-eslint/eslint-plugin": "7.16.1", "@typescript-eslint/parser": "7.16.1", @@ -152,6 +153,8 @@ "svgo": "3.3.2", "syncpack": "12.3.3", "terser": "5.31.3", + "tmp": "0.2.3", + "tree-kill": "1.2.2", "tsm": "2.3.0", "typescript": "5.4.5", "undici": "*", @@ -235,6 +238,7 @@ "serve.debug": "tsm --inspect-brk --conditions=development starters/dev-server.ts 3300", "start": "concurrently \"npm:build.watch\" \"npm:tsc.watch\" -n build,tsc -c green,cyan", "test": "pnpm build.full && pnpm test.unit && pnpm test.e2e", + "test.e2e-cli": "pnpm --filter qwik-cli-e2e e2e", "test.e2e": "pnpm test.e2e.chromium && pnpm test.e2e.webkit", "test.e2e.chromium": "playwright test starters --browser=chromium --config starters/playwright.config.ts", "test.e2e.chromium.debug": "PWDEBUG=1 playwright test starters --browser=chromium --config starters/playwright.config.ts", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 47ab8b172e4..4f543f186e6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -100,6 +100,9 @@ importers: '@types/semver': specifier: 7.5.8 version: 7.5.8 + '@types/tmp': + specifier: 0.2.6 + version: 0.2.6 '@types/which-pm-runs': specifier: 1.0.2 version: 1.0.2 @@ -196,6 +199,12 @@ importers: terser: specifier: 5.31.3 version: 5.31.3 + tmp: + specifier: 0.2.3 + version: 0.2.3 + tree-kill: + specifier: 1.2.2 + version: 1.2.2 tsm: specifier: 2.3.0 version: 2.3.0 @@ -233,6 +242,12 @@ importers: specifier: 3.22.4 version: 3.22.4 + e2e/qwik-cli-e2e: + dependencies: + kleur: + specifier: 4.1.5 + version: 4.1.5 + packages/create-qwik: devDependencies: '@clack/prompts': @@ -3320,6 +3335,9 @@ packages: '@types/set-cookie-parser@2.4.10': resolution: {integrity: sha512-GGmQVGpQWUe5qglJozEjZV/5dyxbOOZ0LHe/lqyWssB88Y4svNfst0uqBVscdDeIKl5Jy5+aPSvy7mI9tYRguw==} + '@types/tmp@0.2.6': + resolution: {integrity: sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==} + '@types/triple-beam@1.3.5': resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} @@ -8957,9 +8975,9 @@ packages: resolution: {integrity: sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==} engines: {node: '>=0.6.0'} - tmp@0.2.1: - resolution: {integrity: sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==} - engines: {node: '>=8.17.0'} + tmp@0.2.3: + resolution: {integrity: sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==} + engines: {node: '>=14.14'} to-fast-properties@2.0.0: resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} @@ -12285,6 +12303,8 @@ snapshots: dependencies: '@types/node': 20.14.11 + '@types/tmp@0.2.6': {} + '@types/triple-beam@1.3.5': {} '@types/unist@2.0.10': {} @@ -19066,15 +19086,13 @@ snapshots: tmp-promise@3.0.3: dependencies: - tmp: 0.2.1 + tmp: 0.2.3 tmp@0.0.33: dependencies: os-tmpdir: 1.0.2 - tmp@0.2.1: - dependencies: - rimraf: 3.0.2 + tmp@0.2.3: {} to-fast-properties@2.0.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index aa6fe414dfc..7ccf3be7cc5 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,5 @@ packages: - 'packages/*' + - 'e2e/*' # - 'starters/adapter/*' # - 'starters/features/*'