test: add retry to validation outlook preview

OfficeDev · Oct 15, 2024 · ac449d1 · ac449d1
1 parent 8897537
commit ac449d1
Show file tree

Hide file tree

Showing 5 changed files with 60 additions and 21 deletions.
diff --git a/packages/tests/src/ui-test/localdebug/localdebug-obo-tab-ts.test.ts b/packages/tests/src/ui-test/localdebug/localdebug-obo-tab-ts.test.ts
@@ -5,7 +5,11 @@
  * @author Kuojian Lu <[email protected]>
  */
 import * as path from "path";
-import { startDebugging, waitForTerminal } from "../../utils/vscodeOperation";
+import {
+  clearNotifications,
+  startDebugging,
+  waitForTerminal,
+} from "../../utils/vscodeOperation";
 import {
   initPage,
   validateReactOutlookTab,
@@ -16,6 +20,7 @@ import {
   Timeout,
   LocalDebugTaskLabel,
   LocalDebugError,
+  LocalDebugTaskResult,
 } from "../../utils/constants";
 import { Env } from "../../utils/env";
 import { it } from "../../utils/it";
@@ -60,12 +65,12 @@ describe("Local Debug M365 Tests", function () {
       try {
         await waitForTerminal(
           LocalDebugTaskLabel.StartBackend,
-          "Worker process started and initialized"
+          LocalDebugTaskResult.FunctionStarted
         );
-
+        await clearNotifications();
         await waitForTerminal(
           LocalDebugTaskLabel.StartFrontend,
-          "Compiled successfully!"
+          LocalDebugTaskResult.FrontendReady
         );
       } catch (error) {
         const errorMsg = error.toString();
@@ -91,10 +96,8 @@ describe("Local Debug M365 Tests", function () {
       await localDebugTestContext.validateLocalStateForTab();
       await validateReactTab(page, Env.displayName, true);
       const m365AppId = await localDebugTestContext.getM365AppId();
-      await page.goto(
-        `https://outlook.office.com/host/${m365AppId}/index?login_hint=${Env.username}`
-      );
-      await validateReactOutlookTab(page, Env.displayName, true);
+      const url = `https://outlook.office.com/host/${m365AppId}/index?login_hint=${Env.username}`;
+      await validateReactOutlookTab(page, url, Env.displayName, true);
     }
   );
 });
diff --git a/packages/tests/src/ui-test/localdebug/localdebug-obo-tab.test.ts b/packages/tests/src/ui-test/localdebug/localdebug-obo-tab.test.ts
@@ -5,7 +5,11 @@
  * @author Kuojian Lu <[email protected]>
  */
 import * as path from "path";
-import { startDebugging, waitForTerminal } from "../../utils/vscodeOperation";
+import {
+  clearNotifications,
+  startDebugging,
+  waitForTerminal,
+} from "../../utils/vscodeOperation";
 import {
   initPage,
   validateReactOutlookTab,
@@ -16,6 +20,7 @@ import {
   Timeout,
   LocalDebugTaskLabel,
   LocalDebugError,
+  LocalDebugTaskResult,
 } from "../../utils/constants";
 import { Env } from "../../utils/env";
 import { it } from "../../utils/it";
@@ -58,12 +63,12 @@ describe("Local Debug M365 Tests", function () {
       try {
         await waitForTerminal(
           LocalDebugTaskLabel.StartBackend,
-          "Worker process started and initialized"
+          LocalDebugTaskResult.FunctionStarted
         );
-
+        await clearNotifications();
         await waitForTerminal(
           LocalDebugTaskLabel.StartFrontend,
-          "Compiled successfully!"
+          LocalDebugTaskResult.FrontendReady
         );
       } catch (error) {
         const errorMsg = error.toString();
@@ -89,10 +94,8 @@ describe("Local Debug M365 Tests", function () {
       await localDebugTestContext.validateLocalStateForTab();
       await validateReactTab(page, Env.displayName, true);
       const m365AppId = await localDebugTestContext.getM365AppId();
-      await page.goto(
-        `https://outlook.office.com/host/${m365AppId}/index?login_hint=${Env.username}`
-      );
-      await validateReactOutlookTab(page, Env.displayName, true);
+      const url = `https://outlook.office.com/host/${m365AppId}/index?login_hint=${Env.username}`;
+      await validateReactOutlookTab(page, url, Env.displayName, true);
     }
   );
 });
diff --git a/packages/tests/src/ui-test/localdebug/localdebug-tab-aad-deploy.test.ts b/packages/tests/src/ui-test/localdebug/localdebug-tab-aad-deploy.test.ts
@@ -1,10 +1,14 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
 /**
  * @author Xiaofu Huang <[email protected]>
  */
 import * as path from "path";
 import * as chai from "chai";
 import { VSBrowser } from "vscode-extension-tester";
 import {
+  clearNotifications,
   getNotification,
   runDeployAadAppManifest,
   startDebugging,
@@ -16,6 +20,8 @@ import {
   Timeout,
   LocalDebugTaskLabel,
   DebugItemSelect,
+  LocalDebugTaskResult,
+  LocalDebugError,
 } from "../../utils/constants";
 import { Env } from "../../utils/env";
 import { it } from "../../utils/it";
@@ -54,10 +60,29 @@ describe("Local Debug Tests", function () {
 
       await startDebugging(DebugItemSelect.DebugInTeamsUsingChrome);
 
-      await waitForTerminal(
-        LocalDebugTaskLabel.StartFrontend,
-        "Compiled successfully!"
-      );
+      try {
+        await waitForTerminal(
+          LocalDebugTaskLabel.StartBackend,
+          LocalDebugTaskResult.FunctionStarted
+        );
+        await clearNotifications();
+        await waitForTerminal(
+          LocalDebugTaskLabel.StartFrontend,
+          LocalDebugTaskResult.FrontendReady
+        );
+      } catch (error) {
+        const errorMsg = error.toString();
+        if (
+          // skip can't find element
+          errorMsg.includes(LocalDebugError.ElementNotInteractableError) ||
+          // skip timeout
+          errorMsg.includes(LocalDebugError.TimeoutError)
+        ) {
+          console.log("[skip error] ", error);
+        } else {
+          chai.expect.fail(errorMsg);
+        }
+      }
 
       await stopDebugging();
 

diff --git a/packages/tests/src/utils/constants.ts b/packages/tests/src/utils/constants.ts
@@ -407,6 +407,7 @@ export enum LocalDebugTaskLabel {
 export class LocalDebugTaskResult {
   static readonly FrontendSuccess = "Compiled successfully";
   static readonly FrontendReady = "ready";
+  static readonly FrontendNoIssue = "webpack compiled";
   static readonly StartSuccess = "started successfully";
   static readonly AzuriteSuccess = "Azurite Table service is successfully";
   static readonly CompiledSuccess = "Found 0 errors";
@@ -419,6 +420,7 @@ export class LocalDebugTaskResult {
   static readonly WebServerSuccess = "press h to show help";
   static readonly DockerFinish = "press any key to close it";
   static readonly DevtunnelSuccess = "Ready to accept connections for tunnel:";
+  static readonly FunctionStarted = "Worker process started and initialized";
 }
 
 export enum LocalDebugTaskLabel2 {

diff --git a/packages/tests/src/utils/playwrightOperation.ts b/packages/tests/src/utils/playwrightOperation.ts
@@ -1040,11 +1040,17 @@ export async function validateReactTab(
 
 export async function validateReactOutlookTab(
   page: Page,
+  url: string,
   displayName: string,
   includeFunction?: boolean
 ) {
-  try {
+  await RetryHandler.retry(async () => {
+    await Promise.all([page.goto(url), page.waitForNavigation()]);
     await page.waitForTimeout(Timeout.longTimeWait);
+    await page.waitForSelector('div[aria-label="hosted-app-tabs"]');
+  }, 3);
+
+  try {
     const frameElementHandle = await page.waitForSelector(
       'iframe[data-tid="app-host-iframe"]'
     );