test(e2e): Switch from Appium to Maestro (#4210)

getsentry · Oct 30, 2024 · 4c3c7db · 4c3c7db
1 parent 0966e69
commit 4c3c7db
Show file tree

Hide file tree

Showing 17 changed files with 178 additions and 383 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -14,6 +14,8 @@ concurrency:
 
 env:
   SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+  MAESTRO_VERSION: '1.39.0'
+  IOS_DEVICE: 'iPhone 14'
 
 jobs:
   diff_check:
@@ -168,12 +170,10 @@ jobs:
             rn-version: '0.76.0'
             runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: ios
             rn-version: '0.65.3'
             runs-on: macos-13
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: android
             runs-on: ubuntu-latest
         exclude:
@@ -308,12 +308,10 @@ jobs:
             rn-version: '0.76.0'
             runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: ios
             rn-version: '0.65.3'
             runs-on: macos-latest
             runtime: 'latest'
-            device: 'iPhone 14'
           - platform: android
             runs-on: ubuntu-latest
         exclude:
@@ -329,12 +327,18 @@ jobs:
           - rn-version: '0.76.0'
             platform: 'ios'
             rn-architecture: 'new'
-    env:
-      PLATFORM: ${{ matrix.platform }}
-      DEVICE: ${{ matrix.device }}
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install Maestro
+        uses: dniHze/maestro-test-action@bda8a93211c86d0a05b7a4597c5ad134566fbde4 # [email protected]
+        with:
+          version: ${{env.MAESTRO_VERSION}}
+
+      - name: Install iDB Companion
+        if: ${{ matrix.platform == 'ios' }}
+        run: brew tap facebook/fb && brew install facebook/fb/idb-companion
+
       - uses: ./.github/actions/disk-cleanup
         if: ${{ matrix.platform == 'android' }}
 
@@ -400,11 +404,10 @@ jobs:
             -timezone US/Pacific
           script: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test
 
-      - uses: actions/cache@v4
+      - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3
         if: ${{ matrix.platform == 'ios' }}
         with:
-          path: test/e2e/DerivedData/Build/Products/Debug-iphonesimulator/WebDriverAgentRunner-Runner.app
-          key: appium-webdriveragent-${{ hashFiles('test/e2e/yarn.lock') }}
+          model: ${{ env.IOS_DEVICE }}
 
       - name: Run tests on iOS
         if: ${{ matrix.platform == 'ios' }}
@@ -415,6 +418,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}-logs
-          path: |
-            test/e2e/*.log
-            test/e2e/*.png
+          path: ./dev-packages/e2e-tests/maestro-logs
diff --git a/dev-packages/e2e-tests/.gitignore b/dev-packages/e2e-tests/.gitignore
@@ -3,4 +3,5 @@
 *.app
 *.apk
 
-react-native-versions
+/react-native-versions
+/maestro-logs
diff --git a/dev-packages/e2e-tests/cli.mjs b/dev-packages/e2e-tests/cli.mjs
@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 'use strict';
 
-import { execSync, spawn } from 'child_process';
+import { execSync, execFileSync, spawn } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
 import { argv, env } from 'process';
@@ -64,8 +64,9 @@ const appRepoDir = `${e2eDir}/react-native-versions/${RNVersion}`;
 const appName = 'RnDiffApp';
 const appDir = `${appRepoDir}/${appName}`;
 const testAppName = `${appName}.${platform == 'ios' ? 'app' : 'apk'}`;
-const runtime = env.IOS_RUNTIME ? env.IOS_RUNTIME : 'latest';
-const device = env.IOS_DEVICE ? env.IOS_DEVICE : 'iPhone 15';
+const testApp = `${e2eDir}/${testAppName}`;
+const appId = platform === 'ios' ? 'org.reactjs.native.example.RnDiffApp' : 'com.rndiffapp';
+const sentryAuthToken = env.SENTRY_AUTH_TOKEN;
 
 // Build and publish the SDK - we only need to do this once in CI.
 // Locally, we may want to get updates from the latest build so do it on every app build.
@@ -143,9 +144,6 @@ if (actions.includes('create')) {
     });
 
     if (fs.existsSync(`${appDir}/Gemfile`)) {
-      // TMP Fix for https://github.com/CocoaPods/Xcodeproj/issues/989
-      fs.appendFileSync(`${appDir}/Gemfile`, "gem 'xcodeproj', '< 1.26.0'\n");
-
       execSync(`bundle install`, { stdio: 'inherit', cwd: appDir, env: env });
       execSync('bundle exec pod install --repo-update', { stdio: 'inherit', cwd: `${appDir}/ios`, env: env });
     } else {
@@ -190,7 +188,8 @@ if (actions.includes('build')) {
                   -workspace ${appName}.xcworkspace \
                   -configuration ${buildType} \
                   -scheme ${appName} \
-                  -destination 'platform=iOS Simulator,OS=${runtime},name=${device}' \
+                  -sdk 'iphonesimulator' \
+                  -destination 'generic/platform=iOS Simulator' \
                   ONLY_ACTIVE_ARCH=yes \
                   -derivedDataPath DerivedData \
                   build | tee xcodebuild.log | xcbeautify`,
@@ -207,123 +206,40 @@ if (actions.includes('build')) {
     appProduct = `${appDir}/android/app/build/outputs/apk/release/app-release.apk`;
   }
 
-  var testApp = `${e2eDir}/${testAppName}`;
   console.log(`Moving ${appProduct} to ${testApp}`);
   if (fs.existsSync(testApp)) fs.rmSync(testApp, { recursive: true });
   fs.renameSync(appProduct, testApp);
 }
 
 if (actions.includes('test')) {
-  if (
-    platform == 'ios' &&
-    !fs.existsSync(`${e2eDir}/DerivedData/Build/Products/Debug-iphonesimulator/WebDriverAgentRunner-Runner.app`)
-  ) {
-    // Build iOS WebDriverAgent
-    execSync(
-      `set -o pipefail && xcodebuild \
-                  -project node_modules/appium-webdriveragent/WebDriverAgent.xcodeproj \
-                  -scheme WebDriverAgentRunner \
-                  -destination 'platform=iOS Simulator,OS=${runtime},name=${device}' \
-                  GCC_TREAT_WARNINGS_AS_ERRORS=0 \
-                  COMPILER_INDEX_STORE_ENABLE=NO \
-                  ONLY_ACTIVE_ARCH=yes \
-                  -derivedDataPath DerivedData \
-                  build | tee xcodebuild-agent.log | xcbeautify`,
-      { stdio: 'inherit', cwd: e2eDir, env: env },
-    );
-  }
-
-  // Start the appium server.
-  var processesToKill = {};
-  async function newProcess(name, process) {
-    await new Promise((resolve, reject) => {
-      process.on('error', e => {
-        console.error(`Failed to start process '${name}': ${e}`);
-        reject(e);
-      });
-      process.on('spawn', () => {
-        console.log(`Process '${name}' (${process.pid}) started`);
-        resolve();
-      });
-    });
-
-    processesToKill[name] = {
-      process: process,
-      complete: new Promise((resolve, _reject) => {
-        process.on('close', resolve);
-      }),
-    };
-  }
-  await newProcess(
-    'appium',
-    spawn('node_modules/.bin/appium', ['--log-timestamp', '--log-no-colors', '--log', `appium${platform}.log`], {
-      stdio: 'inherit',
-      cwd: e2eDir,
-      env: env,
-      shell: false,
-    }),
-  );
-
-  try {
-    await waitForAppium();
-
-    // Run e2e tests
-    const testEnv = env;
-    testEnv.PLATFORM = platform;
-    testEnv.APPIUM_APP = `./${testAppName}`;
-
-    if (platform == 'ios') {
-      testEnv.APPIUM_DERIVED_DATA = 'DerivedData';
-    } else if (platform == 'android') {
-      execSync(`adb devices -l`, { stdio: 'inherit', cwd: e2eDir, env: env });
-
-      execSync(`adb logcat -c`, { stdio: 'inherit', cwd: e2eDir, env: env });
-
-      var adbLogStream = fs.createWriteStream(`${e2eDir}/adb.log`);
-      const adbLogProcess = spawn('adb', ['logcat'], { cwd: e2eDir, env: env, shell: false });
-      adbLogProcess.stdout.pipe(adbLogStream);
-      adbLogProcess.stderr.pipe(adbLogStream);
-      adbLogProcess.on('close', () => adbLogStream.close());
-      await newProcess('adb logcat', adbLogProcess);
-    }
-
-    execSync(`yarn test:e2e:runner --verbose`, { stdio: 'inherit', cwd: e2eDir, env: testEnv });
-  } finally {
-    for (const [name, info] of Object.entries(processesToKill)) {
-      console.log(`Sending termination signal to process '${name}' (${info.process.pid})`);
-
-      // Send SIGTERM first to allow graceful shutdown.
-      info.process.kill(15);
-
-      // Also send SIGKILL after 10 seconds.
-      const killTimeout = setTimeout(() => process.kill(9), '10000');
-
-      // Wait for the process to exit (either via SIGTERM or SIGKILL).
-      const code = await info.complete;
-
-      // Successfully exited now, no need to kill (if it hasn't run yet).
-      clearTimeout(killTimeout);
-
-      console.log(`Process '${name}' (${info.process.pid}) exited with code ${code}`);
+  // Run e2e tests
+  if (platform == 'ios') {
+    try {
+      execSync('xcrun simctl list devices | grep -q "(Booted)"');
+    } catch (error) {
+      throw new Error('No simulator is currently booted. Please boot a simulator before running this script.');
     }
-  }
-}
 
-async function waitForAppium() {
-  console.log('Waiting for Appium server to start...');
-  for (let i = 0; i < 60; i++) {
+    execFileSync('xcrun', ['simctl', 'install', 'booted', testApp]);
+  } else if (platform == 'android') {
     try {
-      await fetch('http://127.0.0.1:4723/sessions', { method: 'HEAD' });
-      console.log('Appium server started');
-      return;
+      execSync('adb devices | grep -q "emulator"');
     } catch (error) {
-      console.log(`Appium server hasn't started yet (${error})...`);
-      await sleep(1000);
+      throw new Error('No Android emulator is currently running. Please start an emulator before running this script.');
     }
+
+    execFileSync('adb', ['install', '-r', '-d', testApp]);
   }
-  throw new Error('Appium server failed to start');
-}
 
-async function sleep(millis) {
-  return new Promise(resolve => setTimeout(resolve, millis));
+  execSync(
+    `maestro test maestro \
+      --env=APP_ID="${appId}" \
+      --env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \
+      --debug-output maestro-logs \
+      --flatten-debug-output`,
+    {
+      stdio: 'inherit',
+      cwd: e2eDir,
+    },
+  );
 }
diff --git a/dev-packages/e2e-tests/maestro/captureException.yml b/dev-packages/e2e-tests/maestro/captureException.yml
@@ -0,0 +1,5 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Capture Exception"
+- runFlow: utils/assertEventIdVisible.yml
diff --git a/dev-packages/e2e-tests/maestro/captureMessage.yml b/dev-packages/e2e-tests/maestro/captureMessage.yml
@@ -0,0 +1,5 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Capture Message"
+- runFlow: utils/assertEventIdVisible.yml
diff --git a/dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml b/dev-packages/e2e-tests/maestro/captureUnhandledPromiseRejection.yml
@@ -0,0 +1,5 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Unhandled Promise Rejection"
+- runFlow: utils/assertEventIdVisible.yml
diff --git a/dev-packages/e2e-tests/maestro/close.yml b/dev-packages/e2e-tests/maestro/close.yml
@@ -0,0 +1,7 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Close"
+
+- assertNotVisible:
+    id: "eventId"
diff --git a/dev-packages/e2e-tests/maestro/crash.yml b/dev-packages/e2e-tests/maestro/crash.yml
@@ -0,0 +1,7 @@
+appId: ${APP_ID}
+---
+- runFlow: utils/launchTestAppClear.yml
+- tapOn: "Crash"
+
+- launchApp
+- assertVisible: "E2E Tests Ready"
diff --git a/dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml b/dev-packages/e2e-tests/maestro/utils/assertEventIdVisible.yml
@@ -0,0 +1,10 @@
+appId: ${APP_ID}
+---
+- extendedWaitUntil:
+    visible:
+      id: "eventId"
+    timeout: 600_000 # 10 minutes
+
+- copyTextFrom:
+    id: "eventId"
+- assertTrue: ${maestro.copiedText}
diff --git a/dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml b/dev-packages/e2e-tests/maestro/utils/launchTestAppClear.yml
@@ -0,0 +1,10 @@
+appId: ${APP_ID}
+---
+- launchApp:
+    clearState: true
+    arguments:
+      sentryAuthToken: ${SENTRY_AUTH_TOKEN}
+
+- extendedWaitUntil:
+    visible: "E2E Tests Ready"
+    timeout: 120_000 # 2 minutes
diff --git a/dev-packages/e2e-tests/package.json b/dev-packages/e2e-tests/package.json
@@ -6,7 +6,8 @@
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc --project tsconfig.build.json",
-    "test:e2e:runner": "NODE_OPTIONS=--experimental-vm-modules jest"
+    "test:ios": "./cli.mjs ios --create --build --test",
+    "test:android": "./cli.mjs android --create --build --test"
   },
   "license": "MIT",
   "devDependencies": {
@@ -29,6 +30,7 @@
   },
   "dependencies": {
     "minimist": "1.2.8",
+    "p-retry": "^6.2.0",
     "semver": "7.6.3",
     "xcode": "3.0.1"
   },