From f43dc91992b215bb6271230b30b792217af918f0 Mon Sep 17 00:00:00 2001 From: Siyuan Chen <67082457+ayachensiyuan@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:19:33 +0800 Subject: [PATCH] test: add docker bot sso sample (#11504) * test: add docker sample --------- Co-authored-by: Ivan_Chen --- .github/workflows/ui-test.yml | 6 ++ packages/tests/README.md | 1 + packages/tests/scripts/randomCases.json | 19 +++- packages/tests/src/ui-test/cliHelper.ts | 78 +++++++++++++++ .../sample-localdebug-bot-sso-docker.test.ts | 35 +++++++ .../sample-remotedebug-bot-sso-docker.test.ts | 30 ++++++ .../src/ui-test/samples/sampleCaseFactory.ts | 44 +++++++-- packages/tests/src/utils/constants.ts | 4 + packages/tests/src/utils/executor.ts | 12 ++- .../tests/src/utils/playwrightOperation.ts | 99 ++++++++++--------- packages/tests/src/utils/vscodeOperation.ts | 2 + 11 files changed, 274 insertions(+), 56 deletions(-) create mode 100644 packages/tests/src/ui-test/samples/sample-localdebug-bot-sso-docker.test.ts create mode 100644 packages/tests/src/ui-test/samples/sample-remotedebug-bot-sso-docker.test.ts diff --git a/.github/workflows/ui-test.yml b/.github/workflows/ui-test.yml index 217af787aa..a258fb6682 100644 --- a/.github/workflows/ui-test.yml +++ b/.github/workflows/ui-test.yml @@ -438,6 +438,12 @@ jobs: run: | npm run build + - name: Install docker extension + if: contains(matrix.test-case, 'docker') + working-directory: packages/tests + run: | + npx extest install-from-marketplace --storage .test-resources --extensions_dir .test-resources --type stable ms-azuretools.vscode-docker + - name: Install vsix(unix) if: matrix.os != 'windows-latest' working-directory: packages/tests diff --git a/packages/tests/README.md b/packages/tests/README.md index 7b60bbc166..86205f8811 100644 --- a/packages/tests/README.md +++ b/packages/tests/README.md @@ -49,6 +49,7 @@ CI_ENABLED=true - (**Required**) Run `npx extest get-chromedriver --storage .test-resources --type stable --code_version 1.88.1` to download chromedriver - (**Required**) Download TeamsFx vsix file to this project root folder. You can download it from the [artifacts of TeamsFx CD action](https://github.com/OfficeDev/TeamsFx/actions/workflows/cd.yml). Remember to unzip. - (**Required**) Run `npx extest install-vsix --storage .test-resources --extensions_dir .test-resources --type stable --vsix_file ${{ YOUR VSIX FILE NAME }} ` to install Teams Toolkit +- (**OPTIONAL**) If local test docker cases, Run `npx extest install-from-marketplace --storage .test-resources --extensions_dir .test-resources --type stable ms-azuretools.vscode-docker` to install docker extension. - (**Required**) Run `npx extest run-tests --storage .test-resources --extensions_dir .test-resources --type stable --code_version 1.88.1 --code_settings ./settings.json ./out/ui-test/**/${{ YOUR TEST CASE }}.test.js` to execute your case - (**OPTIONAL**) If you want to debug your case via vscode, replace "YOUR TEST CASE" with your case name in .vscode/launch.json and click F5 diff --git a/packages/tests/scripts/randomCases.json b/packages/tests/scripts/randomCases.json index 567297f853..0a9c0c47ca 100644 --- a/packages/tests/scripts/randomCases.json +++ b/packages/tests/scripts/randomCases.json @@ -10,7 +10,8 @@ "node-18": [] }, "macos-latest": { - "node-16": [] + "node-16": [], + "node-18": [] } }, "cases": [ @@ -171,5 +172,21 @@ "sample-remotedebug-todo-list-sql", "sample-remotedebug-large-scale-notification" ] + }, + { + "os": { + "ubuntu-latest": { + "node-16": [], + "node-18": [] + }, + "macos-latest": { + "node-16": [], + "node-18": [] + } + }, + "cases": [ + "sample-localdebug-bot-sso-docker", + "sample-remotedebug-bot-sso-docker" + ] } ] \ No newline at end of file diff --git a/packages/tests/src/ui-test/cliHelper.ts b/packages/tests/src/ui-test/cliHelper.ts index f4219ccd16..c2d4a4f63b 100644 --- a/packages/tests/src/ui-test/cliHelper.ts +++ b/packages/tests/src/ui-test/cliHelper.ts @@ -17,6 +17,7 @@ import path from "path"; import * as chai from "chai"; import { Executor } from "../utils/executor"; import * as os from "os"; +import { ChildProcess, ChildProcessWithoutNullStreams } from "child_process"; export class CliHelper { static async addEnv( @@ -615,4 +616,81 @@ export class CliHelper { } console.log("[success] debug successfully !!!"); } + + static async dockerBuild( + projectPath: string, + folder: string, + path = "./", + processEnv: NodeJS.ProcessEnv = process.env, + delay: number = 3 * 60 * 1000 + ): Promise { + console.log(`[start] docker build ... `); + const timeout = timeoutPromise(delay); + const childProcess = spawnCommand( + "docker", + ["build", "-t", folder, path], + { + cwd: projectPath, + env: processEnv ? processEnv : process.env, + }, + (data) => { + console.log(data); + }, + (error) => { + console.log(error); + if (error.includes("Error:")) { + chai.assert.fail(error); + } + } + ); + await Promise.all([timeout, childProcess]); + console.log("[success] docker build successfully !!!"); + return childProcess; + } + + static async dockerRun( + projectPath: string, + folder: string, + processEnv: NodeJS.ProcessEnv = process.env, + delay: number = 30 * 1000 + ): Promise { + console.log(`[start] docker run ... `); + const timeout = timeoutPromise(delay); + const childProcess = spawnCommand( + "docker", + ["run", "-p", "3978:80", "--env-file", ".localConfigs", folder], + { + cwd: projectPath, + env: processEnv ? processEnv : process.env, + }, + (data) => { + console.log(data); + }, + (error) => { + console.log(error); + if (error.includes("Error:")) { + chai.assert.fail(error); + } + } + ); + await Promise.all([timeout, childProcess]); + console.log("[success] docker run successfully !!!"); + return childProcess; + } + + static async stopAllDocker() { + console.log(`[start] docker stop all ... `); + let cmd = ""; + if (os.type() === "Windows_NT") { + cmd = "docker ps -q | ForEach-Object { docker stop $_ }"; + } else { + cmd = "docker stop $(docker ps -q)"; + } + const { stderr, stdout } = await execAsync(cmd); + if (stderr) { + console.log(stderr); + } + console.log(stdout); + console.log("[success] docker stop all successfully !!!"); + } } diff --git a/packages/tests/src/ui-test/samples/sample-localdebug-bot-sso-docker.test.ts b/packages/tests/src/ui-test/samples/sample-localdebug-bot-sso-docker.test.ts new file mode 100644 index 0000000000..702f396256 --- /dev/null +++ b/packages/tests/src/ui-test/samples/sample-localdebug-bot-sso-docker.test.ts @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * @author Ivan Chen + */ + +import { Page } from "playwright"; +import { TemplateProject, LocalDebugTaskLabel } from "../../utils/constants"; +import { validateBot } from "../../utils/playwrightOperation"; +import { CaseFactory } from "./sampleCaseFactory"; +import { Env } from "../../utils/env"; + +class BotSSODockerTestCase extends CaseFactory { + override async onValidate(page: Page): Promise { + return await validateBot(page, { + botCommand: "show", + expected: Env.displayName, + }); + } + public override async onCliValidate(page: Page): Promise { + return await validateBot(page, { + botCommand: "show", + expected: Env.displayName, + }); + } +} + +new BotSSODockerTestCase( + TemplateProject.BotSSODocker, + 26577671, + "v-ivanchen@microsoft.com", + "local", + [LocalDebugTaskLabel.StartLocalTunnel, LocalDebugTaskLabel.DockerRun] +).test(); diff --git a/packages/tests/src/ui-test/samples/sample-remotedebug-bot-sso-docker.test.ts b/packages/tests/src/ui-test/samples/sample-remotedebug-bot-sso-docker.test.ts new file mode 100644 index 0000000000..f87e6a6cce --- /dev/null +++ b/packages/tests/src/ui-test/samples/sample-remotedebug-bot-sso-docker.test.ts @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * @author Ivan Chen + */ + +import { Page } from "playwright"; +import { TemplateProject, LocalDebugTaskLabel } from "../../utils/constants"; +import { validateBot } from "../../utils/playwrightOperation"; +import { CaseFactory } from "./sampleCaseFactory"; +import { Env } from "../../utils/env"; + +class BotSSODockerTestCase extends CaseFactory { + override async onValidate(page: Page): Promise { + return await validateBot(page, { + botCommand: "show", + expected: Env.displayName, + }); + } +} + +new BotSSODockerTestCase( + TemplateProject.BotSSODocker, + 27852471, + "v-ivanchen@microsoft.com", + "dev", + undefined, + { container: true } +).test(); diff --git a/packages/tests/src/ui-test/samples/sampleCaseFactory.ts b/packages/tests/src/ui-test/samples/sampleCaseFactory.ts index da261b7980..a4c8a2b530 100644 --- a/packages/tests/src/ui-test/samples/sampleCaseFactory.ts +++ b/packages/tests/src/ui-test/samples/sampleCaseFactory.ts @@ -32,6 +32,7 @@ import path from "path"; import { Executor } from "../../utils/executor"; import { ChildProcessWithoutNullStreams } from "child_process"; import { initDebugPort } from "../../utils/commonUtils"; +import { CliHelper } from "../cliHelper"; const debugMap: Record Promise> = { [LocalDebugTaskLabel.StartFrontend]: async () => { @@ -97,6 +98,12 @@ const debugMap: Record Promise> = { LocalDebugTaskResult.WebServerSuccess ); }, + [LocalDebugTaskLabel.DockerRun]: async () => { + await waitForTerminal( + LocalDebugTaskLabel.DockerRun, + LocalDebugTaskResult.WebServerSuccess + ); + }, }; export abstract class CaseFactory { @@ -118,6 +125,8 @@ export abstract class CaseFactory { debug?: "cli" | "ttk"; botFlag?: boolean; repoPath?: string; + container?: boolean; + dockerFolder?: string; }; public constructor( @@ -139,6 +148,8 @@ export abstract class CaseFactory { debug?: "cli" | "ttk"; botFlag?: boolean; repoPath?: string; + container?: boolean; + dockerFolder?: string; } = {} ) { this.sampleName = sampleName; @@ -277,6 +288,7 @@ export abstract class CaseFactory { let azSqlHelper: AzSqlHelper | undefined; let devtunnelProcess: ChildProcessWithoutNullStreams; let debugProcess: ChildProcessWithoutNullStreams; + let dockerProcess: ChildProcessWithoutNullStreams; let successFlag = true; let envContent = ""; let botFlag = false; @@ -363,6 +375,9 @@ export abstract class CaseFactory { sampledebugContext.appName, sampledebugContext.projectPath ); + if (options?.container) { + await Executor.login(); + } await sampledebugContext.deployProject( sampledebugContext.projectPath, Timeout.botDeploy @@ -392,11 +407,23 @@ export abstract class CaseFactory { "local" ); expect(provisionSuccess).to.be.true; - const { success: deploySuccess } = await Executor.deploy( - sampledebugContext.projectPath, - "local" - ); - expect(deploySuccess).to.be.true; + if (!options.container) { + const { success: deploySuccess } = await Executor.deploy( + sampledebugContext.projectPath, + "local" + ); + expect(deploySuccess).to.be.true; + } else { + await CliHelper.dockerBuild( + sampledebugContext.projectPath, + options.dockerFolder || "" + ); + + dockerProcess = await CliHelper.dockerRun( + sampledebugContext.projectPath, + options.dockerFolder || "" + ); + } const teamsAppId = await sampledebugContext.getTeamsAppId(env); expect(teamsAppId).to.not.be.empty; @@ -421,7 +448,8 @@ export abstract class CaseFactory { successFlag = false; expect.fail(errorMsg); } - } + }, + options.container ); await new Promise((resolve) => setTimeout(resolve, 2 * 60 * 1000) @@ -458,6 +486,10 @@ export abstract class CaseFactory { // kill process await Executor.closeProcess(debugProcess); if (botFlag) await Executor.closeProcess(devtunnelProcess); + if (dockerProcess) { + await Executor.closeProcess(dockerProcess); + await CliHelper.stopAllDocker(); + } await initDebugPort(); } diff --git a/packages/tests/src/utils/constants.ts b/packages/tests/src/utils/constants.ts index 1bc2303353..096a8fedeb 100644 --- a/packages/tests/src/utils/constants.ts +++ b/packages/tests/src/utils/constants.ts @@ -57,6 +57,7 @@ export enum TemplateProject { RetailDashboard = "Contoso Retail Dashboard", TabSSOApimProxy = "SSO Enabled Tab via APIM Proxy", LargeScaleBot = "Large Scale Notification Bot", + BotSSODocker = "Containerized Bot App with SSO Enabled", } export enum TemplateProjectFolder { @@ -128,6 +129,7 @@ export const sampleProjectMap: Record = [TemplateProject.RetailDashboard]: TemplateProjectFolder.RetailDashboard, [TemplateProject.TabSSOApimProxy]: TemplateProjectFolder.TabSSOApimProxy, [TemplateProject.LargeScaleBot]: TemplateProjectFolder.LargeScaleBot, + [TemplateProject.BotSSODocker]: TemplateProjectFolder.BotSSODocker, }; export enum Resource { @@ -380,6 +382,7 @@ export enum LocalDebugTaskLabel { Azurite = "Start Azurite emulator", Compile = "Compile typescript", StartWebServer = "Start web server", + DockerRun = "docker-run: debug", } export class LocalDebugTaskResult { @@ -394,6 +397,7 @@ export class LocalDebugTaskResult { static readonly Error = "error"; static readonly DebuggerAttached = "Debugger attached"; static readonly WebServerSuccess = "press h to show help"; + static readonly DockerRunFinish = "press any key to close it"; } export enum LocalDebugTaskLabel2 { diff --git a/packages/tests/src/utils/executor.ts b/packages/tests/src/utils/executor.ts index e8c506cc3f..f206c045d8 100644 --- a/packages/tests/src/utils/executor.ts +++ b/packages/tests/src/utils/executor.ts @@ -197,7 +197,8 @@ export class Executor { v3 = true, processEnv: NodeJS.ProcessEnv = process.env, onData?: (data: string) => void, - onError?: (data: string) => void + onError?: (data: string) => void, + openOnly?: boolean ) { console.log(`[start] ${env} debug ... `); const childProcess = spawn( @@ -208,7 +209,12 @@ export class Executor { : v3 ? "teamsapp" : "teamsfx", - ["preview", v3 ? "--env" : "", v3 ? `${env}` : `--${env}`], + [ + "preview", + v3 ? "--env" : "", + v3 ? `${env}` : `--${env}`, + openOnly ? "--open-only" : "", + ], { cwd: projectPath, env: processEnv ? processEnv : process.env, @@ -594,6 +600,8 @@ export class Executor { } catch (error) { console.log(error); } + } else { + console.log(childProcess); } } } diff --git a/packages/tests/src/utils/playwrightOperation.ts b/packages/tests/src/utils/playwrightOperation.ts index 085c453719..868ea7cada 100644 --- a/packages/tests/src/utils/playwrightOperation.ts +++ b/packages/tests/src/utils/playwrightOperation.ts @@ -101,6 +101,9 @@ export const debugInitMap: Record Promise> = { [TemplateProject.LargeScaleBot]: async () => { await startDebugging(); }, + [TemplateProject.BotSSODocker]: async () => { + await startDebugging("Debug in Docker (Chrome)"); + }, }; export async function initPage( @@ -135,25 +138,24 @@ export async function initPage( page.click("input.button[type='submit']"), page.waitForNavigation(), ]); - }); - - // input password - console.log(`fill in password`); - await page.fill("input.input[type='password'][name='passwd']", password); + // input password + console.log(`fill in password`); + await page.fill("input.input[type='password'][name='passwd']", password); - // sign in - await Promise.all([ - page.click("input.button[type='submit']"), - page.waitForNavigation(), - ]); + // sign in + await Promise.all([ + page.click("input.button[type='submit']"), + page.waitForNavigation(), + ]); - // stay signed in confirm page - console.log(`stay signed confirm`); - await Promise.all([ - page.click("input.button[type='submit'][value='Yes']"), - page.waitForNavigation(), - ]); - await page.waitForTimeout(Timeout.shortTimeLoading); + // stay signed in confirm page + console.log(`stay signed confirm`); + await Promise.all([ + page.click("input.button[type='submit'][value='Yes']"), + page.waitForNavigation(), + ]); + await page.waitForTimeout(Timeout.shortTimeLoading); + }); // add app await RetryHandler.retry(async (retries: number) => { @@ -1431,17 +1433,17 @@ export async function validateBot( console.log("no message to dismiss"); } - if (options?.botCommand === "show") { - try { - console.log("sending message ", options?.botCommand); - await executeBotSuggestionCommand(page, frame, options?.botCommand); - await frame?.click('button[name="send"]'); - } catch (e: any) { - console.log( - `[Command "${options?.botCommand}" not executed successfully] ${e.message}` - ); - } - await RetryHandler.retry(async () => { + await RetryHandler.retry(async () => { + if (options?.botCommand === "show") { + try { + console.log("sending message ", options?.botCommand); + await executeBotSuggestionCommand(page, frame, options?.botCommand); + await frame?.click('button[name="send"]'); + } catch (e: any) { + console.log( + `[Command "${options?.botCommand}" not executed successfully] ${e.message}` + ); + } try { // wait for alert message to show const btn = await frame?.waitForSelector( @@ -1473,31 +1475,34 @@ export async function validateBot( await popup.click("input.button[type='submit'][value='Accept']"); } } catch (error) { + console.log(error); + // reopen skip login + await frame?.waitForSelector(`p:has-text("${options?.expected}")`); console.log("reopen skip step"); + console.log("verify bot successfully!!!"); + await page.waitForTimeout(Timeout.shortTimeLoading); + return; } + await frame?.waitForSelector(`p:has-text("${options?.expected}")`); + console.log("verify bot successfully!!!"); + console.log(`${options?.expected}`); + } else { await RetryHandler.retry(async () => { - await frame?.waitForSelector(`p:has-text("${options?.expected}")`); + console.log("sending message ", options?.botCommand); + await executeBotSuggestionCommand( + page, + frame, + options?.botCommand || "welcome" + ); + await frame?.click('button[name="send"]'); + await frame?.waitForSelector( + `p:has-text("${options?.expected || ValidationContent.Bot}")` + ); console.log("verify bot successfully!!!"); }, 2); console.log(`${options?.expected}`); - }, 2); - console.log(`${options?.expected}`); - } else { - await RetryHandler.retry(async () => { - console.log("sending message ", options?.botCommand); - await executeBotSuggestionCommand( - page, - frame, - options?.botCommand || "welcome" - ); - await frame?.click('button[name="send"]'); - await frame?.waitForSelector( - `p:has-text("${options?.expected || ValidationContent.Bot}")` - ); - console.log("verify bot successfully!!!"); - }, 2); - console.log(`${options?.expected}`); - } + } + }, 2); await page.waitForTimeout(Timeout.shortTimeLoading); } catch (error) { await page.screenshot({ diff --git a/packages/tests/src/utils/vscodeOperation.ts b/packages/tests/src/utils/vscodeOperation.ts index 3b00d1da05..bdfe335ef3 100644 --- a/packages/tests/src/utils/vscodeOperation.ts +++ b/packages/tests/src/utils/vscodeOperation.ts @@ -220,6 +220,7 @@ export async function execCommandIfExist( ): Promise { const driver = VSBrowser.instance.driver; await VSBrowser.instance.waitForWorkbench(); + console.log("[start] run vsc command: ", commandName); if (os.type() === "Darwin") { // command + P await driver.actions().keyDown(Key.COMMAND).keyDown("P").perform(); @@ -245,6 +246,7 @@ export async function execCommandIfExist( if (timeout) { await driver.sleep(timeout); } + console.log("[finish] run vsc command successfully"); return; } }