diff --git a/README.md b/README.md index 1159a8e8..eba51830 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,11 @@ With the hosted version: 2. Setup a PostgreSQL instance (version 15 minimum) 3. Copy the content of `packages/backend/.env.example` to `packages/backend/.env` and fill the missing values 4. Copy the content of `packages/frontend/.env.example` to `packages/backend/.env` -5. Run `npm install` -6. Run `npm run migrate:db` -7. Run `npm run dev` +6. Run `npm install` +7. Run `npm run migrate:db` +8. Run `npm run dev` +8. Run `npm run test` for run test playwright +8. Run `npm run test:debug` for run test debug playwright You can now open the dashboard at `http://localhost:8080`. When using our JS or Python SDK, you need to set the environment variable `LUNARY_API_URL` to `http://localhost:3333`. You can use `LUNARY_VERBOSE=True` to see all the event sent by the SDK diff --git a/e2e/feebback.spec.ts b/e2e/feebback.spec.ts new file mode 100644 index 00000000..48fbbabc --- /dev/null +++ b/e2e/feebback.spec.ts @@ -0,0 +1,227 @@ +import { Page, expect, test } from "@playwright/test" +require('dotenv').config(); +import { config , UserInfo} from "./helpers/constants" +import { uniqueStr } from "./helpers/uniqueStr" +import lunary from "lunary" +import OpenAI from "openai" +import { monitorOpenAI } from "lunary/openai" +import { setOrgPro } from "./utils/db" +import { LoginPage } from "./page/login.po" +import { CommAction } from "./page/common.po" +import { HomePage } from "./page/home.po" +import { SignUpPage } from "./page/signUp.po" +import { AnalyticsPage } from "./page/analytics.po" +import { LogsPage } from "./page/logs.po" +import { ThreatsPage } from "./page/threats.po"; +import { TracesPage } from "./page/traces.po"; + +let page: Page; +let context: any; +let userName:string; +let email :string; +let email1 :string; +let email2 :string; +let userName1 :string; +let userName2 :string; + +let contentSystem1 :string; +let contentSystem2 :string; +let contentUser1 :string; +let contentUser2 :string; + +test.describe('Feedback', () => { + test.beforeAll(async ({ browser }) => { + context = await browser.newContext(); + page = await context.newPage(); + const login = new LoginPage(page); + + const homePage = new HomePage(page); + const signUpPage = new SignUpPage(page); + const commonPage = new CommAction(page); + const analyticsPage = new AnalyticsPage(page); + + userName =uniqueStr('user'); + email = uniqueStr('test') + '@example.com'; + + email1 = uniqueStr('test1') + '@example.com'; + userName1 =uniqueStr('user1'); + + email2 = uniqueStr('test2') + '@example.com'; + userName2 =uniqueStr('user2'); + + contentSystem1 = "Hi friend message 1"; + contentSystem2 = "Hi friend message 2"; + contentUser1 = "Can you help me on message 1"; + contentUser2 = "Can you help me on message 2"; + await login.openUrl(config.BASE_URL); + await homePage.clickSignUpBtn(); + await signUpPage.signUpAccountStep1(email, UserInfo.password, userName); + await signUpPage.signUpAccountStep2("TESTPROJECT","TESTORG", UserInfo.companySize, UserInfo.option); + await signUpPage.clickSkipDashboard(); + await commonPage.clickMenu('Analytics'); + const projectId = await analyticsPage.getProjectId(); + await commonPage.initLunaryOpenAI(projectId); + await setOrgPro() + const openai = monitorOpenAI(new OpenAI({ apiKey: config.OPENAI_API_KEY})) + const thread = lunary.openThread({ + userId: "tristina Test", + userProps: { name: "tristina Testing" }, + tags: ['testing thread'] + }) + const msgId = thread.trackMessage({ + role: 'assistant', + content: 'testing' + }) + const result = await openai.chat.completions.create({ + model: "gpt-4o", + temperature: 0.9, + tags: ["chat", "support"], // Optional: tags + user: email1, // Optional: user ID + userProps: { name: userName1 }, // Optional: user properties + messages: [ + { role: "system", content: contentSystem1 }, + { role: "user", content: contentUser1 }, + ], + + }).setParent(msgId) + + + const msgId2 = thread.trackMessage({ + role: 'user', + content: result.choices[0].message.content + }) + + const result2 = await openai.chat.completions.create({ + model: "gpt-4o", + temperature: 0.9, + tags: ["chat", "support"], // Optional: tags + user: email2, // Optional: user ID + userProps: { name: userName2 }, // Optional: user properties + messages: [ + { role: "system", content: contentSystem2}, + { role: "user", content: contentUser2}, + ], + + }).setParent(msgId2) + + const agent = lunary.wrapAgent(function ChatbotAgent(query) { + console.log('test agent'); + }) + + await agent("Hello!").setParent(msgId) + const calculator = lunary.wrapTool(async function Calculator(input) { + // Your custom logic + // ... + }) + + await calculator('1 + 2') + }); + + + + test("Adding a thumb up/down to a from a llm call to verify feedback", async ({page}) => { + const commonPage = new CommAction(page); + const logsPage = new LogsPage(page); + const loginPage = new LoginPage(page); + + await loginPage.openUrl(config.BASE_URL); + await loginPage.login(email, UserInfo.password); + + await commonPage.clickMenu('Logs'); + + await logsPage.clickMessage(contentUser2); + await logsPage.verifyThumbUpDownIsDisplayedOnBanner(); + await logsPage.clickThumbUpIcon(); + await logsPage.verifyThumbUpIconTurnGreen(); + await logsPage.closeModal(); + await logsPage.verifyThumbUpIconIsDisplayed(contentUser2); + + await logsPage.clickMessage(contentUser2); + await logsPage.clickThumbDownIcon(); + await logsPage.verifyThumbDownIconTurnRed(); + await logsPage.closeModal(); + await logsPage.verifyThumbDownIconIsDisplayed(contentUser2); + + }) + + test("Adding a comment and verify it", async ({page}) => { + const commonPage = new CommAction(page); + const logsPage = new LogsPage(page); + const loginPage = new LoginPage(page); + + await loginPage.openUrl(config.BASE_URL); + await loginPage.login(email, UserInfo.password); + + await commonPage.clickMenu('Logs'); + + await logsPage.clickMessage(contentUser2); + await logsPage.clickMessageIcon(); + const comment =uniqueStr('user'); + await logsPage.sendComment(comment); + await logsPage.closeModal(); + await logsPage.hoverCommentIcon(contentUser2); + await logsPage.verifyCommentIsDisplayed(contentUser2, comment); + }) + + test("Adding a thumb up/down from a trace to verify feedback", async ({page}) => { + const commonPage = new CommAction(page); + const logsPage = new LogsPage(page); + const threatsPage = new ThreatsPage(page); + const tracesPage = new TracesPage(page); + const loginPage = new LoginPage(page); + + await loginPage.openUrl(config.BASE_URL); + await loginPage.login(email, UserInfo.password); + + await commonPage.clickMenu('Logs'); + await commonPage.clickTab('Threads'); + await threatsPage.clickTagName('testing thread'); + await threatsPage.clickViewTraceButton(); + await tracesPage.clickTagName('gpt-4o'); + await tracesPage.verifyThumbUpDownIsDisplayedOnBanner(); + await tracesPage.clickThumbUpIcon(); + + await tracesPage.verifyThumbUpIconTurnGreen(); + + await commonPage.clickMenu('Logs'); + await logsPage.verifyThumbUpIconIsDisplayed(contentUser1); + + await commonPage.clickTab('Threads'); + await threatsPage.clickTagName('testing thread'); + await threatsPage.clickViewTraceButton(); + await tracesPage.clickTagName('gpt-4o'); + await tracesPage.clickThumbDownIcon(); + await tracesPage.verifyThumbDownIconTurnRed(); + + await commonPage.clickMenu('Logs'); + await logsPage.verifyThumbDownIconIsDisplayed(contentUser1); + + + }) + + test("Adding a comment from a trace to verify feedback", async ({page}) => { + const commonPage = new CommAction(page); + const logsPage = new LogsPage(page); + const loginPage = new LoginPage(page); + const threatsPage = new ThreatsPage(page); + const tracesPage = new TracesPage(page); + const comment =uniqueStr('comment from trace'); + await loginPage.openUrl(config.BASE_URL+''); + await loginPage.login(email, UserInfo.password); + + await commonPage.clickMenu('Logs'); + + await commonPage.clickTab('Threads'); + await threatsPage.clickTagName('testing thread'); + await threatsPage.clickViewTraceButton(); + await tracesPage.clickTagName('gpt-4o'); + await tracesPage.clickMessageIcon(); + await tracesPage.sendComment(comment); + + await commonPage.clickMenu('Logs'); + await logsPage.hoverCommentIcon(contentUser1); + await logsPage.verifyCommentIsDisplayed(contentUser1, comment); + + + }) +}); \ No newline at end of file diff --git a/e2e/helpers/constants.ts b/e2e/helpers/constants.ts new file mode 100644 index 00000000..b863c004 --- /dev/null +++ b/e2e/helpers/constants.ts @@ -0,0 +1,26 @@ +const BASE_URL = process.env.APP_URL || 'http://localhost:8080'; +const OPENAI_API_KEY = process.env.OPENAI_API_KEY; +const API_URL = process.env.LUNARY_API_URL || 'http://localhost:3333'; +if ( + !OPENAI_API_KEY +) { + throw 'Missing environment variables'; +} +export const config = { + BASE_URL, + OPENAI_API_KEY, + API_URL, + }; + + export const Constants = { + defaultPassword: 'bl@ckr0ck', + defaultuserName: 'bl@automationTesting@gmail.com' + }; + + export const UserInfo = { + userName: 'Tristina', + email: 'test@example.com', + password: '12345678', + companySize: '6-49', + option: 'Other', + }; diff --git a/e2e/helpers/uniqueStr.ts b/e2e/helpers/uniqueStr.ts new file mode 100644 index 00000000..dc426001 --- /dev/null +++ b/e2e/helpers/uniqueStr.ts @@ -0,0 +1,29 @@ + export function generateCode(length: number) { + let result = ''; + const characters = '0123456789'; + const charactersLength = characters.length; + let counter = 0; + while (counter < length) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + counter += 1; + } + return result; + } + + export function generateUserName(length: number) { + let result = ''; + const characters = '0123456789'; + const charactersLength = characters.length; + let counter = 0; + while (counter < length) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + counter += 1; + } + result = 'test'+result; + return result; + } + + export function uniqueStr(s: string) { + const now = new Date().getTime(); + return `${s}${now}`; + } \ No newline at end of file diff --git a/e2e/page/analytics.po.ts b/e2e/page/analytics.po.ts new file mode 100644 index 00000000..270b594b --- /dev/null +++ b/e2e/page/analytics.po.ts @@ -0,0 +1,17 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class AnalyticsPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async getProjectId(): Promise { + return await this.page.locator('//code').innerText(); + } + + +} diff --git a/e2e/page/common.po.ts b/e2e/page/common.po.ts new file mode 100644 index 00000000..98a9dbe1 --- /dev/null +++ b/e2e/page/common.po.ts @@ -0,0 +1,140 @@ +import { Locator, Page, expect } from '@playwright/test'; +import lunary from "lunary" +import OpenAI from "openai" +require('dotenv').config(); +import { config } from "../helpers/constants" +import { setOrgPro } from "../utils/db" +import { monitorOpenAI } from "lunary/openai" + +export class CommAction{ + readonly page: Page; + + constructor(page: Page) { + this.page = page; + } + + async openUrl(urlPath: string): Promise { + await this.page.goto(urlPath, { timeout: 90 * 1000 }); + } + + async clickMenu(menuName: string): Promise { + await this.page.waitForLoadState("networkidle") + await this.page.getByRole('link', { name: menuName}).click(); + } + + async waitSomeSeconds(second: number): Promise { + await this.page.waitForTimeout(second*1000); + } + + + async clickTab(tabName: string): Promise { + await this.page.getByText(tabName).click(); + } + + async initLunaryOpenAI(projectId: string): Promise { + lunary.init({ + "appId": projectId, // Your unique app ID obtained from the dashboard + "apiUrl": config.API_URL, // Optional: Use a custom endpoint if you're self-hosting (you can also set LUNARY_API_URL) + "verbose": true // Optional: Enable verbose logging for debugging + }) + } + + async openAI(email: string, userName:string): Promise { + const openai = monitorOpenAI(new OpenAI({ apiKey: config.OPENAI_API_KEY})) + const result = await openai.chat.completions.create({ + model: "gpt-4o", + temperature: 0.9, + tags: ["chat", "support"], // Optional: tags + user: email, // Optional: user ID + userProps: { name: userName }, // Optional: user properties + messages: [ + { role: "system", content: "You are an helpful assistant" }, + { role: "user", content: "Hello friend" }, + ], + }) + } + async verifyThumbUpDownIsDisplayedOnBanner(): Promise { + await expect(this.page.locator("//*[contains(@class,'thumb-up')and contains(@fill,'color-gray')]").first()).toBeVisible() + await expect(this.page.locator("//*[contains(@class,'thumb-down') and contains(@fill,'color-gray')]").first()).toBeVisible() + } + + async clickMessage(message: string): Promise { + await this.page.getByText(message).click(); + } + + async clickThumbUpIcon(): Promise { + await this.page.locator("//*[contains(@class,'thumb-up')and contains(@fill,'color-gray')]").last().click() + } + + async clickMessageIcon(): Promise { + await this.page.locator("//*[contains(@class,'tabler-icon-message') and contains(@stroke,'gray')]").click() + } + + async sendComment(content:string): Promise { + await this.page.getByPlaceholder('Add a comment').fill(content); + await this.page.locator('[role="dialog"] span.mantine-Button-label').filter({hasText: 'Save'}).click(); + } + + async hoverCommentIcon(message:string): Promise { + const messageLocator = this.page.locator("table tr").filter({hasText: message}) + .locator(`//*[contains(@fill,'color-teal-5')]`); + if(await messageLocator.isVisible() == false) + { + await this.waitSomeSeconds(5); + await this.page.reload(); + } + await messageLocator.hover(); + } + + async verifyCommentIsDisplayed(message:string, content:string): Promise { + const messageLocator = await this.page.locator('div.mantine-Tooltip-tooltip').innerText(); + expect(messageLocator).toContain(content); + } + + async verifyThumbUpIconTurnGreen(): Promise { + await expect(this.page.locator("//*[contains(@class,'thumb-up') and contains(@fill,'color-green')]").last()).toBeVisible() + } + + async verifyThumbDownIconTurnRed(): Promise { + await expect(this.page.locator("//*[contains(@class,'thumb-down') and contains(@fill,'color-red')]").last()).toBeVisible() + } + + async verifyThumbUpIconIsDisplayed(message:string): Promise { + const feedback = this.page.locator("table tr").filter({hasText: message}) + .locator(`//*[contains(@class,'thumb-up') and contains(@fill,'color-green')]`); + console.log("check feedback is visible : "+await feedback.isVisible()) + if(await feedback.isVisible() == false) + { + await this.waitSomeSeconds(5); + await this.page.reload(); + } + await expect(feedback).toBeVisible(); + + } + + async verifyThumbDownIconIsDisplayed(message:string): Promise { + const feedbackRed = this.page.locator("table tr").filter({hasText: message}) + .locator(`//*[contains(@class,'thumb-down') and contains(@fill,'color-red')]`); + if(await feedbackRed.isVisible() == false){ + await this.waitSomeSeconds(5); + await this.page.reload(); + } + await expect(feedbackRed).toBeVisible(); + + } + + async clickThumbDownIcon(): Promise { + await this.page.locator("//*[contains(@class,'thumb-down')and contains(@fill,'color-gray')]").last().click() + } + + async logOut(): Promise { + await this.page.getByTestId("account-sidebar-item").click() + await this.page.getByTestId("logout-button").click() + } + + async verifyCurrentUrl(url: string): Promise { + await this.page.waitForURL(url); + expect(await this.page.url()).toContain(url); + } + +} diff --git a/e2e/page/dataset.po.ts b/e2e/page/dataset.po.ts new file mode 100644 index 00000000..0b031190 --- /dev/null +++ b/e2e/page/dataset.po.ts @@ -0,0 +1,63 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class DatasetPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async clickNewChatDatasetBtn(): Promise { + await this.page.getByRole('button', { name: 'New Dataset' }).click(); + await this.page.getByRole('menuitem', { name: 'New Chat Dataset (OpenAI' }).click(); + } + + async clickNewDatasetBtn(): Promise { + await this.page.getByRole('button', { name: 'New Dataset' }).click(); + await this.page.getByRole('menuitem', { name: 'New Text Dataset' }).click(); + } + + async renameDataset(name: string): Promise { + await this.page.waitForTimeout(1000); + await this.page.locator("//*[contains(@class,'tabler-icon-pencil')]").first().click(); + await this.page.getByTestId('rename-input').waitFor({state: "visible"}); + await this.page.getByTestId('rename-input').fill(name); + await this.page.keyboard.press('Enter'); + } + + async createChatDataset(name: string): Promise { + await this.page.waitForTimeout(1000); + await this.page.locator("//*[contains(@class,'tabler-icon-pencil')]").first().click(); + await this.page.getByTestId('rename-input').waitFor({state: "visible"}); + await this.page.getByTestId('rename-input').fill(name); + await this.page.keyboard.press('Enter'); + } + + async verifyDatasetNameIsDisplayed(name: string): Promise { + await this.page.locator('h3.mantine-Title-root').filter({hasText: name}).waitFor({timeout: 60000}); + await expect(this.page.locator('h3.mantine-Title-root').filter({hasText: name})).toBeVisible(); + } + + async verifyDatasetNameIsNotDisplayed(name: string): Promise { + await expect(this.page.locator('h3.mantine-Title-root').filter({hasText: name})).toBeHidden(); + } + + async deleteDatasetIcon(name: string): Promise { + await this.page.locator(`//h3[contains(text(),'${name}')]/ancestor::div[contains(@class,'mantine-Card-root')]//button`).first().click(); + await this.page.getByRole('heading', { name: 'Please confirm your action' }).click(); + await this.page.getByText('Are you sure you want to').click(); + await this.page.getByRole('button', { name: 'Confirm' }).click(); + } + + async clickBackBtn(): Promise { + await this.page.getByRole('link', { name: '← Back' }).click(); + } + + async inputPrompt(content: string): Promise { + await this.page.getByTestId('prompt-text-editor').fill(content); + } + + +} diff --git a/e2e/page/home.po.ts b/e2e/page/home.po.ts new file mode 100644 index 00000000..546dcc15 --- /dev/null +++ b/e2e/page/home.po.ts @@ -0,0 +1,17 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class HomePage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async clickSignUpBtn(): Promise { + await this.page.getByText('Sign Up').click(); + } + + +} diff --git a/e2e/page/login.po.ts b/e2e/page/login.po.ts new file mode 100644 index 00000000..38cf7d33 --- /dev/null +++ b/e2e/page/login.po.ts @@ -0,0 +1,19 @@ +import { Locator, Page } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class LoginPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async login(userName: string, password: string): Promise { + await this.page.locator('input[type="email"]').fill(userName); + await this.page.locator('span.mantine-Button-label').filter({hasText: 'Continue'}).click(); + await this.page.locator('input[type="password"]').fill(password); + await this.page.locator('span.mantine-Button-label').filter({hasText: 'Login'}).click(); + await this.page.locator('span.mantine-Button-label').filter({hasText: 'Login'}).waitFor({state: 'detached', timeout: 30000}); + } +} diff --git a/e2e/page/logs.po.ts b/e2e/page/logs.po.ts new file mode 100644 index 00000000..5bb66f82 --- /dev/null +++ b/e2e/page/logs.po.ts @@ -0,0 +1,19 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class LogsPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async closeModal(): Promise { + // await this.page.locator(`[role="dialog"] button[data-variant="subtle"]`).click() + await this.page.getByRole('banner').getByRole('button').click(); + } + + + +} diff --git a/e2e/page/requestResetPassword.po.ts b/e2e/page/requestResetPassword.po.ts new file mode 100644 index 00000000..09d8d3af --- /dev/null +++ b/e2e/page/requestResetPassword.po.ts @@ -0,0 +1,16 @@ +import { Locator, Page } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class RequestResetPasswordPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async inputEmail(email: string): Promise { + await this.page.getByPlaceholder('Your email').fill(email); + await this.page.locator(`span.mantine-Button-label`).filter({hasText: 'Submit'}).click(); + } +} diff --git a/e2e/page/resetPassword.po.ts b/e2e/page/resetPassword.po.ts new file mode 100644 index 00000000..8800fb63 --- /dev/null +++ b/e2e/page/resetPassword.po.ts @@ -0,0 +1,16 @@ +import { Locator, Page } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class ResetPasswordPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async inputNewPassword(newPass: string): Promise { + await this.page.getByPlaceholder('Your new password').fill(newPass); + await this.page.locator('span.mantine-Button-label').filter({hasText: 'Submit'}).click(); + } +} diff --git a/e2e/page/signUp.po.ts b/e2e/page/signUp.po.ts new file mode 100644 index 00000000..230a40c5 --- /dev/null +++ b/e2e/page/signUp.po.ts @@ -0,0 +1,50 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class SignUpPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async signUpAccountStep1(email: string, password: string, userName:string): Promise { + await this.page.waitForLoadState("networkidle") + await this.page.getByPlaceholder('Your email').click(); + await this.page.getByPlaceholder('Your email').fill(email); + await this.page.getByPlaceholder('Your full name').click(); + await this.page.getByPlaceholder('Your full name').fill(userName); + await this.page.getByPlaceholder('Your password').click(); + await this.page.getByPlaceholder('Your password').fill(password); + await this.page.getByRole('button', { name: 'Continue →' }).click(); + } + + async signUpAccountStep2(projectName: string, organization: string, companySize: string, option: string): Promise { + await this.page.getByPlaceholder("Your project name").click() + await this.page.getByPlaceholder("Your project name").fill(projectName) + + await this.page.getByPlaceholder("Organization name").click() + await this.page.getByPlaceholder("Organization name").fill(organization) + await this.page.getByLabel(companySize).check(); + await this.page.getByPlaceholder('Select an option').click(); + await this.page.getByRole('option', { name: option }).click(); + await this.page.getByRole('button', { name: 'Create account' }).click(); + } + + async verifySignUpSuccess(): Promise { + expect( await this.page.getByRole('heading', { name: 'You\'re all set 🎉' })).toBeVisible(); + } + + async openDashboard(): Promise { + await this.page.getByRole('button', { name: 'Open Dashboard' }).click(); + } + + async clickSkipDashboard(): Promise { + await this.page.waitForNavigation() + + await expect(this.page.getByText("Are you free in the next days")).toBeVisible() + await this.page.getByRole("button", { name: "Skip to Dashboard" }).click() + } + +} diff --git a/e2e/page/threats.po.ts b/e2e/page/threats.po.ts new file mode 100644 index 00000000..718d9a9d --- /dev/null +++ b/e2e/page/threats.po.ts @@ -0,0 +1,21 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class ThreatsPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + async clickTagName(tag : string): Promise { + await this.page.getByText(tag).click(); + } + + async clickViewTraceButton(): Promise { + await this.page.getByRole('button', { name: 'View trace' }).click(); + await this.page.waitForLoadState("networkidle") + } + + +} diff --git a/e2e/page/traces.po.ts b/e2e/page/traces.po.ts new file mode 100644 index 00000000..4d7c2a0a --- /dev/null +++ b/e2e/page/traces.po.ts @@ -0,0 +1,16 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class TracesPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + async clickTagName(tag : string): Promise { + await this.page.getByText(tag).first().click(); + } + + +} diff --git a/e2e/page/users.po.ts b/e2e/page/users.po.ts new file mode 100644 index 00000000..50e339d0 --- /dev/null +++ b/e2e/page/users.po.ts @@ -0,0 +1,19 @@ +import { Locator, Page } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class UserPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async searchUser(userName: string): Promise { + await this.page.locator('#search').fill(userName); + } + + async clickUserName(userName: string): Promise { + await this.page.locator('table p').filter({hasText: userName}).click(); + } +} diff --git a/e2e/page/usersDetail.po.ts b/e2e/page/usersDetail.po.ts new file mode 100644 index 00000000..53d399ed --- /dev/null +++ b/e2e/page/usersDetail.po.ts @@ -0,0 +1,18 @@ +import { Locator, Page, expect } from '@playwright/test'; +import { CommAction } from './common.po'; + +export class UserDetailPage extends CommAction { + readonly page: Page; + + constructor(page: Page) { + super(page); + this.page = page; + } + + async verifyUserInfo(userName: string, email: string): Promise { + await expect( this.page.locator('h4').filter({hasText: userName})).toBeVisible(); + await expect( this.page.getByRole('code').filter({hasText: email})).toBeVisible(); + } + + +} diff --git a/e2e/resetPassword.spec.ts b/e2e/resetPassword.spec.ts new file mode 100644 index 00000000..5f93c22b --- /dev/null +++ b/e2e/resetPassword.spec.ts @@ -0,0 +1,52 @@ +import test, { Page,expect } from "@playwright/test" +import { RequestResetPasswordPage } from "./page/requestResetPassword.po" +import { getRecoveryToken } from "./utils/db"; +import { config,UserInfo } from "./helpers/constants"; +import { CommAction } from "./page/common.po"; +import { ResetPasswordPage } from "./page/resetPassword.po"; +import {LoginPage} from "./page/login.po"; +import { uniqueStr } from "./helpers/uniqueStr" +import { SignUpPage } from "./page/signUp.po" +import { HomePage } from "./page/home.po" + +let page: Page; +let context: any; +let userName:string; +let email :string; +test.describe('Feedback', () => { + test.beforeAll(async ({ browser }) => { + context = await browser.newContext(); + page = await context.newPage(); + const login = new LoginPage(page); + + const homePage = new HomePage(page); + const signUpPage = new SignUpPage(page); + const commonPage = new CommAction(page); + + userName =uniqueStr('user'); + email = uniqueStr('test') + '@example.com'; + + await login.openUrl(config.BASE_URL); + await homePage.clickSignUpBtn(); + await signUpPage.signUpAccountStep1(email, UserInfo.password, userName); + await signUpPage.signUpAccountStep2("TESTPROJECT","TESTORG", UserInfo.companySize, UserInfo.option); + await signUpPage.clickSkipDashboard(); + + }); + +test("Reset password", async ({ }) => { +const requestResetPass = new RequestResetPasswordPage(page); + const commonPage = new CommAction(page); + const resetPasswordPage = new ResetPasswordPage(page); + const loginPage = new LoginPage(page); + await commonPage.logOut(); + await requestResetPass.openUrl(config.BASE_URL + '/request-password-reset') + await requestResetPass.inputEmail(email); + const token = await getRecoveryToken(email); + await requestResetPass.openUrl(config.BASE_URL + `/reset-password?token=${token}`); + await resetPasswordPage.inputNewPassword('Abcd1234!'); + await commonPage.logOut(); + await loginPage.login(email,'Abcd1234!'); + await commonPage.verifyCurrentUrl(config.BASE_URL+'/analytics'); +}) +}); \ No newline at end of file diff --git a/e2e/smokeTest.spec.ts b/e2e/smokeTest.spec.ts new file mode 100644 index 00000000..f5111d93 --- /dev/null +++ b/e2e/smokeTest.spec.ts @@ -0,0 +1,76 @@ +import { Page, expect, test } from "@playwright/test" +import { UserInfo, config } from "./helpers/constants" +require('dotenv').config(); +import { generateUserName, uniqueStr } from "./helpers/uniqueStr" +import {SignUpPage} from "../e2e/page/signUp.po"; +import { CommAction } from "./page/common.po"; +import { DatasetPage } from "./page/dataset.po"; +import {LoginPage} from "./page/login.po"; +import { UserPage } from "./page/users.po"; +import { UserDetailPage } from "./page/usersDetail.po"; +import { HomePage } from "./page/home.po"; +import { AnalyticsPage } from "./page/analytics.po"; + +let page: Page; +let context: any; +let userName:string; +let email :string; + +test.describe('Smoke test', () => { + test.describe.configure({ mode: 'serial' }); + test.beforeAll(async ({ browser }) => { + context = await browser.newContext(); + page = await context.newPage(); + const login = new LoginPage(page); + userName =uniqueStr('test'); + email = uniqueStr('test') + '@example.com'; + await login.openUrl(config.BASE_URL); + }); + + test("Verify search user", async () => { + const homePage = new HomePage(page); + const signUpPage = new SignUpPage(page); + const commonPage = new CommAction(page); + const analyticsPage = new AnalyticsPage(page); + const userPage = new UserPage(page); + const userDetailPage = new UserDetailPage(page); + + await homePage.clickSignUpBtn(); + await signUpPage.signUpAccountStep1(email, UserInfo.password, userName); + await signUpPage.signUpAccountStep2("TESTPROJECT","TESTORG", UserInfo.companySize, UserInfo.option); + await signUpPage.clickSkipDashboard(); + await commonPage.clickMenu('Analytics'); + const projectId = await analyticsPage.getProjectId(); + await commonPage.initLunaryOpenAI(projectId); + await commonPage.openAI(email, userName); + + await commonPage.clickMenu('Users'); + await userPage.searchUser(userName); + await userPage.clickUserName(userName); + await userDetailPage.verifyUserInfo(userName, email); + }) + + test("Verify dataset", async () => { + const commonPage = new CommAction(page); + const datasetPage = new DatasetPage(page); + const loginPage = new LoginPage(page); + + await commonPage.clickMenu('Evaluations'); + await commonPage.clickMenu('Datasets'); + await datasetPage.clickNewChatDatasetBtn(); + const dataset1 = generateUserName(10); + await datasetPage.renameDataset(dataset1); + await datasetPage.clickBackBtn(); + await datasetPage.verifyDatasetNameIsDisplayed(dataset1); + + await datasetPage.clickNewDatasetBtn(); + const dataset2 = generateUserName(10); + await datasetPage.renameDataset(dataset2); + await datasetPage.clickBackBtn(); + await datasetPage.verifyDatasetNameIsDisplayed(dataset2); + await datasetPage.deleteDatasetIcon(dataset2); + await datasetPage.verifyDatasetNameIsNotDisplayed(dataset2); + }) +}); + + diff --git a/e2e/utils/db.ts b/e2e/utils/db.ts index a16b9c1c..8fac9fb1 100644 --- a/e2e/utils/db.ts +++ b/e2e/utils/db.ts @@ -52,3 +52,9 @@ export async function populateLogs() { ] await sql`insert into run ${sql(logs)}` } + + export async function getRecoveryToken(email: string) { + const [account] = await sql` + select recovery_token from account where email = ${email} ` + return account.recoveryToken; +} diff --git a/package.json b/package.json index dacd3d1b..a79ed83f 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "dev:radar": "npm -w packages/backend run dev:radar", "dev:withradar": "concurrently \"npm run dev:frontend\" \"npm run dev:backend\" \"npm run start:ml\" \"npm run dev:radar\"", "test": "npx playwright test", + "test:debug": "npx playwright test --debug", "test:ui": "npx playwright test --ui", "postinstall": "npx patch-package" }, diff --git a/packages/backend/.env.example b/packages/backend/.env.example index a18af13f..b0bb6d0b 100644 --- a/packages/backend/.env.example +++ b/packages/backend/.env.example @@ -8,3 +8,4 @@ LUNARY_PUBLIC_KEY=259d2d94-9446-478a-ae04-484de705b522 OPENAI_API_KEY=sk-... OPENROUTER_API_KEY=sk-... PALM_API_KEY=AI... +LUNARY_API_URL= diff --git a/playwright.config.ts b/playwright.config.ts index b2b74736..b26e211c 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -21,21 +21,30 @@ export default defineConfig({ /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, + retries: 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: "html", // reporter: process.env.CI ? "github" : "html", /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + timeout: 360000, + expect: { + /** + * Maximum time expect() should wait for the condition to be met. + * For example in `await expect(locator).toHaveText();` + */ + timeout: 5000 + }, use: { actionTimeout: 10 * 1000, - navigationTimeout: 10 * 1000, + navigationTimeout: 30 * 1000, // Uses Vercel deployment URL in CI, otherwise uses localhost. baseURL: process.env.CI ? process.env.BASE_URL : "http://127.0.0.1:8080", permissions: ["clipboard-read", "clipboard-write"], video: "retain-on-failure", - + + screenshot: 'only-on-failure', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: "on-first-retry", },