From 0c02b2b6b4e73c803604d9d25aa45bc2b13af8cb Mon Sep 17 00:00:00 2001 From: Tzviel Solodar Date: Sun, 26 Jan 2025 18:26:52 +0200 Subject: [PATCH 1/4] feat: stabilize playwright driver + add screen name to pilot's steps + add logs to file --- src/Copilot.test.ts | 58 +- src/actions/PilotPerformer.test.ts | 25 +- src/actions/PilotPerformer.ts | 23 +- src/drivers/playwright/index.ts | 566 ++++++++++++++++-- src/integration tests/index.test.ts | 2 + src/types/framework.ts | 2 + src/types/pilot.ts | 4 + src/utils/PilotPromptCreator.test.ts | 11 +- src/utils/PilotPromptCreator.ts | 16 +- .../PilotPromptCreator.test.ts.snap | 84 ++- src/utils/extractOutputs.ts | 1 + src/utils/logger/index.ts | 64 +- 12 files changed, 714 insertions(+), 142 deletions(-) diff --git a/src/Copilot.test.ts b/src/Copilot.test.ts index 4f45e13..a72fa78 100644 --- a/src/Copilot.test.ts +++ b/src/Copilot.test.ts @@ -6,6 +6,7 @@ import { ScreenCapturerResult, PromptHandler, PilotStepReport, + PilotReport, } from "@/types"; import { mockCache, mockedCacheFile } from "./test-utils/cache"; import { ScreenCapturer } from "@/utils/ScreenCapturer"; @@ -298,12 +299,11 @@ describe("Copilot", () => { instance.extendAPICatalog([barCategory1]); instance.extendAPICatalog([barCategory2], dummyContext); - expect(mockConfig.frameworkDriver.apiCatalog.categories.length).toEqual( - 1, - ); - expect(mockConfig.frameworkDriver.apiCatalog.categories[0].items).toEqual( - [...barCategory1.items, ...barCategory2.items], - ); + expect(mockConfig.frameworkDriver.apiCatalog.categories.length).toEqual(1); + expect(mockConfig.frameworkDriver.apiCatalog.categories[0].items).toEqual([ + ...barCategory1.items, + ...barCategory2.items, + ]); expect(spyCopilotStepPerformer).toHaveBeenCalledWith(dummyContext); }); @@ -326,9 +326,11 @@ describe("Copilot", () => { const goal = "test goal"; const mockPilotResult = { + summary: "Test completed successfully", goal, steps: [ { + screenName: "Screen 1", plan: { thoughts: "Step 1 thoughts", action: "Tap on GREAT button", @@ -337,6 +339,7 @@ describe("Copilot", () => { goalAchieved: false, }, { + screenName: "Screen 2", plan: { thoughts: "Completed successfully", action: "success", @@ -362,9 +365,10 @@ describe("Copilot", () => { const goal = "Test the login flow"; const pilotOutputStep1: PilotStepReport = { + screenName: "Login Screen", plan: { thoughts: "Step 1 thoughts", - action: "Tap on GREAT button", + action: "Tap on Login button", }, review: { ux: { @@ -382,8 +386,9 @@ describe("Copilot", () => { }; const pilotOutputSuccess: PilotStepReport = { + screenName: "Home Screen", plan: { - thoughts: "Completed successfully all was good ", + thoughts: "Completed successfully", action: "success", }, review: { @@ -399,38 +404,43 @@ describe("Copilot", () => { }, }, goalAchieved: true, + summary: "All was good", }; - jest.spyOn(instance["pilotPerformer"], "perform").mockResolvedValue({ - summary: "all was good", - goal: goal, - steps: [ - { - plan: pilotOutputStep1.plan, - code: "code executed", - review: pilotOutputStep1.review, - goalAchieved: true, - }, - ], - review: pilotOutputSuccess.review, - }); + jest + .spyOn(instance["pilotPerformer"], "perform") + .mockResolvedValue({ + summary: pilotOutputSuccess.summary, + goal: goal, + steps: [ + { + screenName: pilotOutputStep1.screenName, + plan: pilotOutputStep1.plan, + code: "code executed", + review: pilotOutputStep1.review, + goalAchieved: pilotOutputStep1.goalAchieved, + }, + ], + review: pilotOutputSuccess.review, + }); const result = await instance.pilot(goal); expect(instance["pilotPerformer"].perform).toHaveBeenCalledWith(goal); expect(result).toEqual({ - summary: "all was good", + summary: pilotOutputSuccess.summary, goal: goal, steps: [ { + screenName: pilotOutputStep1.screenName, plan: pilotOutputStep1.plan, code: "code executed", review: pilotOutputStep1.review, - goalAchieved: true, + goalAchieved: pilotOutputStep1.goalAchieved, }, ], review: pilotOutputSuccess.review, }); }); }); -}); +}); \ No newline at end of file diff --git a/src/actions/PilotPerformer.test.ts b/src/actions/PilotPerformer.test.ts index 68f8361..482ac5a 100644 --- a/src/actions/PilotPerformer.test.ts +++ b/src/actions/PilotPerformer.test.ts @@ -2,7 +2,7 @@ import { PilotPerformer } from "@/actions/PilotPerformer"; import { PilotPromptCreator } from "@/utils/PilotPromptCreator"; import { ScreenCapturer } from "@/utils/ScreenCapturer"; import { - PreviousStep, + PilotPreviousStep, PromptHandler, ScreenCapturerResult, PilotStepReport, @@ -14,8 +14,11 @@ const GOAL = "tap button"; const VIEW_HIERARCHY = ""; const GENERATED_PROMPT = "generated prompt"; -// Updated PROMPT_RESULT to include UX and Accessibility sections +// Updated PROMPT_RESULT to include screenName, UX, and Accessibility sections const PROMPT_RESULT = ` + +default name + I think this is great @@ -79,12 +82,12 @@ describe("PilotPerformer", () => { capture: jest.fn(), } as unknown as jest.Mocked; - // Instantiate PilotPerformer with the mocks, including the capture function + // Instantiate PilotPerformer with the mocks pilotPerformer = new PilotPerformer( mockPromptCreator, mockCopilotStepPerformer, mockPromptHandler, - mockScreenCapturer, // Pass the mock capture function + mockScreenCapturer, ); }); @@ -125,6 +128,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { + screenName: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -167,6 +171,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { + screenName: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -209,6 +214,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { + screenName: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -243,11 +249,10 @@ describe("PilotPerformer", () => { it("should perform an intent successfully with previous intents", async () => { const intent = "current intent"; - const previousIntents: PreviousStep[] = [ + const previousIntents: PilotPreviousStep[] = [ { + screenName: "default", step: "previous intent", - code: "previous code", - result: "previous result", }, ]; @@ -260,6 +265,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { + screenName: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -295,6 +301,7 @@ describe("PilotPerformer", () => { describe("perform", () => { it("should perform multiple steps until success is returned", async () => { const pilotOutputStep1: PilotStepReport = { + screenName: "Screen 1", plan: { thoughts: "Step 1 thoughts", action: "Tap on GREAT button", @@ -315,6 +322,7 @@ describe("PilotPerformer", () => { }; const pilotOutputSuccess: PilotStepReport = { + screenName: "Screen 2", plan: { thoughts: "Completed successfully", action: "success", @@ -365,6 +373,7 @@ describe("PilotPerformer", () => { goal: GOAL, steps: [ { + screenName: "Screen 1", plan: pilotOutputStep1.plan, code: "code executed", review: pilotOutputStep1.review, @@ -377,4 +386,4 @@ describe("PilotPerformer", () => { expect(result).toEqual(expectedReport); }); }); -}); +}); \ No newline at end of file diff --git a/src/actions/PilotPerformer.ts b/src/actions/PilotPerformer.ts index 5e48956..e6a3275 100644 --- a/src/actions/PilotPerformer.ts +++ b/src/actions/PilotPerformer.ts @@ -14,6 +14,7 @@ import { import { extractOutputs, OUTPUTS_MAPPINGS } from "@/utils/extractOutputs"; import { CopilotStepPerformer } from "@/actions/CopilotStepPerformer"; import { ScreenCapturer } from "@/utils/ScreenCapturer"; +import fs from 'fs'; import logger from "@/utils/logger"; export class PilotPerformer { @@ -96,7 +97,7 @@ export class PilotPerformer { const generatedPilotTaskDetails: string = await this.promptHandler.runPrompt(prompt, snapshot); - const { thoughts, action, ux, a11y } = extractOutputs({ + const { screenName, thoughts, action, ux, a11y } = extractOutputs({ text: generatedPilotTaskDetails, outputsMapper: OUTPUTS_MAPPINGS.PILOT_STEP, }); @@ -106,12 +107,18 @@ export class PilotPerformer { isBold: true, color: "whiteBright", }); - + const plan: PilotStepPlan = { action, thoughts }; const review: PilotReview = { ux: this.extractReviewOutput(ux), a11y: this.extractReviewOutput(a11y), }; + + logger.info({ + message: `Conducting review for ${screenName}\n`, + isBold: true, + color: 'whiteBright', + }); review.ux && this.logReviewSection(review.ux, "ux"); review.a11y && this.logReviewSection(review.a11y, "a11y"); @@ -125,7 +132,7 @@ export class PilotPerformer { }).summary : undefined; - return { plan, review, goalAchieved, summary }; + return { screenName, plan, review, goalAchieved, summary }; } catch (error) { analysisLoggerSpinner.stop( "failure", @@ -157,7 +164,7 @@ export class PilotPerformer { for (let step = 0; step < maxSteps; step++) { const screenCapture: ScreenCapturerResult = await this.screenCapturer.capture(); - const { plan, review, goalAchieved, summary } = + const { screenName, plan, review, goalAchieved, summary } = await this.analyseScreenAndCreateCopilotStep( goal, previousSteps, @@ -170,19 +177,20 @@ export class PilotPerformer { isBold: true, color: "whiteBright", }); + logger.writeLogsToFile(`pilot_logs_${Date.now()}`); return { goal, summary, steps: [...report.steps], review }; } - + const { code, result } = await this.copilotStepPerformer.perform( plan.action, [...copilotSteps], screenCapture, ); - copilotSteps = [...copilotSteps, { step: plan.action, code, result }]; - previousSteps = [...previousSteps, { step: plan.action, review }]; + previousSteps = [...previousSteps, { screenName, step: plan.action, review }]; const stepReport: PilotStepReport = { + screenName, plan, review, code, @@ -195,6 +203,7 @@ export class PilotPerformer { logger.warn( `🛬 Pilot finished execution due to limit of ${maxSteps} steps has been reached`, ); + logger.writeLogsToFile(`pilot_logs_${Date.now()}`); return report; } } diff --git a/src/drivers/playwright/index.ts b/src/drivers/playwright/index.ts index c5c2fa3..fc7e5c1 100644 --- a/src/drivers/playwright/index.ts +++ b/src/drivers/playwright/index.ts @@ -82,6 +82,7 @@ export class PlaywrightFrameworkDriver implements TestingFrameworkDriver { playwright, expect: playwrightExpect, }, + restrictions:['Never use expect on the page it self for example : await expect(page).toBeVisible()'], categories: [ { title: "Page Management", @@ -90,7 +91,7 @@ export class PlaywrightFrameworkDriver implements TestingFrameworkDriver { signature: "getCurrentPage(): playwright.Page | undefined", description: "Gets the current active page instance.", example: "const page = getCurrentPage();", - guidelines: [ + guidelines: [ "Always check if page exists before operations.", "Returns undefined if no page is set.", "Use before any page interactions.", @@ -101,7 +102,7 @@ export class PlaywrightFrameworkDriver implements TestingFrameworkDriver { description: "Sets the current active page for interactions.", example: "const page = await context.newPage(); setCurrentPage(page);", - guidelines: [ + guidelines: [ "Must be called after creating a new page.", "Required before any page interactions.", "Only one page can be active at a time.", @@ -122,8 +123,11 @@ export class PlaywrightFrameworkDriver implements TestingFrameworkDriver { }); const context = await browser.newContext(); const page = await context.newPage(); -setCurrentPage(page);`, - guidelines: [ +setCurrentPage(page); +await page.goto('https://www.test.com/'); +await page.waitForLoadState('load') +`, + guidelines: [ "Set longer timeouts (30s or more) to handle slow operations.", "Can use chromium, firefox, or webkit browsers.", "Remember to call setCurrentPage after creating a page.", @@ -139,7 +143,11 @@ setCurrentPage(page);`, actionTimeout: 15000 // Action specific timeout }); const page = await context.newPage(); -setCurrentPage(page);`, +setCurrentPage(page); +await page.goto('https://www.test.com/'); +await page.waitForLoadState('load') +`, + guidelines: [ "Each context is isolated with separate cookies/localStorage.", "Set specific timeouts for different operation types.", @@ -158,9 +166,10 @@ setCurrentPage(page);`, if (page) { await page.goto('https://example.com'); // Verify navigation success using assertions - await expect(page.getByRole('heading')).toBeVisible(); + await page.waitForLoadState('load') }`, guidelines: [ + "Use only await page.waitForLoadState('load') after page.goto(). never use expect()", "Always verify navigation success with assertions.", "Avoid using waitUntil options - use assertions instead.", "Set proper timeouts at browser/context level.", @@ -172,11 +181,8 @@ if (page) { example: `const page = getCurrentPage(); if (page) { await page.reload(); - // Verify reload success using assertions - await expect(page.getByRole('main')).toBeVisible(); }`, - guidelines: [ - "Use assertions to verify page state after reload.", + guidelines: [ "Avoid explicit waits - let assertions handle timing.", "Good for refreshing stale content.", ], @@ -193,7 +199,7 @@ if (page) { if (page) { await page.getByRole('button', { name: 'Submit' }).click(); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Preferred way to locate interactive elements.", "Improves test accessibility coverage.", @@ -206,7 +212,7 @@ if (page) { if (page) { await page.getByText('Welcome').isVisible(); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Good for finding visible text on page.", "Can use exact or fuzzy matching.", @@ -219,7 +225,7 @@ if (page) { if (page) { await page.getByLabel('Username').fill('john'); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Best practice for form inputs.", "More reliable than selectors.", @@ -237,7 +243,7 @@ if (page) { if (page) { await page.getByRole('button').click(); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Automatically waits for element.", "Handles scrolling automatically.", @@ -250,94 +256,528 @@ if (page) { if (page) { await page.getByLabel('Password').fill('secret'); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Preferred over type() for forms.", "Clears existing value first.", ], }, + { + signature: "await page.goBack([options])", + description: "Navigates back in the browser history.", + example: `const page = getCurrentPage(); + if (page) { + await page.goBack(); + }`, + guidelines: [ + "Simulates clicking the browser's back button.", + "Waits for the navigation to complete.", + "Options can modify waiting behavior.", + "Useful for testing multi-page flows.", + ], + }, + { + signature: "await page.dragAndDrop(source, target, [options])", + description: "Drags an element to a target element or position.", + example: `const page = getCurrentPage(); + if (page) { + await page.dragAndDrop('#item', '#destination'); + }`, + guidelines: [ + "Source and target can be selectors or locators.", + "Automatically waits for elements to be visible and enabled.", + "Handles mouse events to simulate drag-and-drop.", + "Options can control the delay and force of the action.", + ], + }, + { + signature: "await locator.check([options])", + description: "Checks a checkbox or radio button.", + example: `const page = getCurrentPage(); + if (page) { + await page.getByLabel('Accept Terms').check(); + }`, + guidelines: [ + "Use for input elements of type checkbox or radio.", + "Automatically waits for the element to be visible and enabled.", + "Ensures the element is checked; does nothing if already checked.", + "Handles any necessary scrolling into view.", + ], + }, + { + signature: "await locator.uncheck([options])", + description: "Unchecks a checkbox.", + example: `const page = getCurrentPage(); + if (page) { + await page.getByLabel('Subscribe to newsletter').uncheck(); + }`, + guidelines: [ + "Use for input elements of type checkbox.", + "Automatically waits for the element to be visible and enabled.", + "Ensures the element is unchecked; does nothing if already unchecked.", + "Handles any necessary scrolling into view.", + ], + }, + { + signature: "await locator.scrollIntoViewIfNeeded()", + description: "Scrolls the element into view if it is not already visible.", + example: `const page = getCurrentPage(); + if (page) { + await page.getByText('Load More').scrollIntoViewIfNeeded(); + }`, + guidelines: [ + "Use when the element might be outside the viewport.", + "Automatically waits for the element to be in the DOM.", + "Useful before performing actions that require the element to be in view.", + ], + }, ], }, { - title: "Assertions", + title: "State Checks", items: [ { - signature: "await expect(locator).toBeVisible()", - description: - "Asserts element is visible using Playwright assertions.", + signature: "await page.waitForLoadState('load')", + description: "Waits for the page to fully load.", example: `const page = getCurrentPage(); -if (page) { - await expect(page.getByText('Success')).toBeVisible(); -}`, + if (page) { + await page.goto('https://www.wix.com/domains'); + await page.waitForLoadState('load'); + }`, guidelines: [ - "Uses Playwright's built-in assertions with auto-retry.", - "More reliable than isVisible() for assertions.", - "Has built-in timeout and retry logic.", + "Waits until the 'load' event is fired.", + "Ensures the page is fully loaded before proceeding.", + "Useful when automatic waiting is insufficient.", ], }, { - signature: "await expect(locator).toHaveText(text)", - description: - "Asserts element's text content using Playwright assertions.", + signature: "const currentURL = page.url()", + description: "Gets the current URL of the page.", example: `const page = getCurrentPage(); -if (page) { - await expect(page.getByRole('heading')).toHaveText('Welcome'); -}`, + if (page) { + const currentURL = page.url(); + console.log('Current URL:', currentURL); + }`, guidelines: [ - "Uses Playwright's built-in assertions with auto-retry.", - "Can check exact or partial text.", - "More reliable than textContent() for assertions.", + "Returns the current URL as a string synchronously.", + "Useful for logging or conditional logic.", + "No need to use `await` since it's a synchronous method.", ], }, { - signature: "await expect(locator).toHaveCount(number)", - description: "Asserts number of matching elements.", + signature: "const title = await page.title()", + description: "Gets the page title.", example: `const page = getCurrentPage(); -if (page) { - await expect(page.getByRole('listitem')).toHaveCount(3); -}`, + if (page) { + const title = await page.title(); + console.log('Page Title:', title); + }`, guidelines: [ - "Uses Playwright's built-in assertions with auto-retry.", - "Good for checking element counts.", - "More reliable than count() for assertions.", + "Returns the current page title.", + "Use `await` to get the title asynchronously.", + "Auto-waits for the title to be available.", + ], + }, + { + signature: "const isVisible = await locator.isVisible()", + description: "Checks if an element is visible.", + example: `const page = getCurrentPage(); + if (page) { + const isVisible = await page.getByText('Sign In').isVisible(); + if (isVisible) { + // Proceed with sign-in process + } + }`, + guidelines: [ + "Returns a boolean value immediately.", + "Good for conditional logic and flow control.", + "Does not auto-wait or retry", + ], + }, + { + signature: "const isEnabled = await locator.isEnabled()", + description: "Checks if an element is enabled.", + example: `const page = getCurrentPage(); + if (page) { + const isEnabled = await page.getByRole('button', { name: 'Submit' }).isEnabled(); + if (isEnabled) { + // Click the submit button + await page.getByRole('button', { name: 'Submit' }).click(); + } + }`, + guidelines: [ + "Returns a boolean indicating if the element is enabled.", + "Useful to check if a button or input is interactive.", + "Good for conditional actions based on element state.", + ], + }, + { + signature: "const isChecked = await locator.isChecked()", + description: "Checks if a checkbox or radio button is checked.", + example: `const page = getCurrentPage(); + if (page) { + const isChecked = await page.getByLabel('Accept Terms').isChecked(); + if (!isChecked) { + await page.getByLabel('Accept Terms').check(); + } + }`, + guidelines: [ + "Returns a boolean indicating if the element is checked.", + "Applicable to checkboxes and radio buttons.", + "Good for ensuring the desired state before proceeding.", + ], + }, + { + signature: "const isDisabled = await locator.isDisabled()", + description: "Checks if an element is disabled.", + example: `const page = getCurrentPage(); + if (page) { + const isDisabled = await page.getByRole('button', { name: 'Submit' }).isDisabled(); + if (isDisabled) { + console.log('Submit button is disabled'); + } + }`, + guidelines: [ + "Returns a boolean indicating if the element is disabled.", + "Useful for checking if an element is not interactive.", + "Can inform flow control based on element availability.", + ], + }, + { + signature: "const isEditable = await locator.isEditable()", + description: "Checks if an input element is editable.", + example: `const page = getCurrentPage(); + if (page) { + const isEditable = await page.getByPlaceholder('Enter your name').isEditable(); + if (isEditable) { + await page.fill('input[placeholder="Enter your name"]', 'John Doe'); + } + }`, + guidelines: [ + "Returns a boolean indicating if the element can be edited.", + "Applicable to input, textarea, and contenteditable elements.", + "Good for ensuring the field is ready for input.", + ], + }, + { + signature: "const isHidden = await locator.isHidden()", + description: "Checks if an element is hidden.", + example: `const page = getCurrentPage(); + if (page) { + const isHidden = await page.getByText('Loading...').isHidden(); + if (isHidden) { + // Proceed since the loading indicator is gone + } + }`, + guidelines: [ + "Returns a boolean value immediately.", + "Useful for conditional flow based on element invisibility.", + "Does not auto-wait or retry; use `expect()` for assertions.", + ], + }, + { + signature: "const isDetached = await locator.isDetached()", + description: "Checks if an element is detached from the DOM.", + example: `const page = getCurrentPage(); + if (page) { + const modal = page.getByRole('dialog'); + // Perform some action that closes the modal + await page.getByRole('button', { name: 'Close' }).click(); + const isDetached = await modal.isDetached(); + if (isDetached) { + // Modal has been removed from the DOM + } + }`, + guidelines: [ + "Returns true if the element is not attached to the DOM.", + "Useful to confirm that an element has been completely removed.", + "Can inform subsequent actions that depend on DOM structure.", + ], + }, + { + signature: "const textContent = await locator.textContent()", + description: "Retrieves the text content of an element.", + example: `const page = getCurrentPage(); + if (page) { + const message = await page.getByTestId('welcome-message').textContent(); + console.log('Welcome message:', message); + }`, + guidelines: [ + "Returns the text inside the element.", + "Useful for extracting text for further processing.", + "Consider using `expect().toHaveText()` for assertions.", + ], + }, + { + signature: "const value = await locator.inputValue()", + description: "Retrieves the current value of an input field.", + example: `const page = getCurrentPage(); + if (page) { + const email = await page.getByPlaceholder('Email').inputValue(); + console.log('Entered email:', email); + }`, + guidelines: [ + "Returns the value property of input and textarea elements.", + "Useful for validating or using the input value in logic.", + "Consider using `expect().toHaveValue()` for assertions.", + ], + }, + { + signature: "const elementHandles = await locator.elementHandles()", + description: "Gets all matching element handles.", + example: `const page = getCurrentPage(); + if (page) { + const items = await page.getByRole('listitem').elementHandles(); + console.log('Number of items:', items.length); + }`, + guidelines: [ + "Retrieves an array of element handles matching the locator.", + "Useful for performing actions on multiple elements.", + "Remember to dispose of handles if necessary to prevent memory leaks.", + ], + }, + { + signature: "await page.waitForTimeout(timeout)", + description: "Waits for a specified amount of time.", + example: `const page = getCurrentPage(); + if (page) { + // Wait for 2 seconds + await page.waitForTimeout(2000); + }`, + guidelines: [ + "Pauses execution for the given time in milliseconds.", + "Useful for debugging or waiting in non-deterministic cases.", + "Prefer explicit waits for events or conditions when possible.", ], }, ], }, { - title: "State Checks", + title: "Assertions", items: [ { - signature: "await locator.isVisible()", - description: "Checks if element is visible.", + signature: "await expect(locator).toBeVisible()", + description: "Asserts that the element is visible.", example: `const page = getCurrentPage(); -if (page) { - if (await page.getByText('Error').isVisible()) { - // handle error state - } -}`, + if (page) { + await expect(page.getByText('Welcome')).toBeVisible(); + }`, guidelines: [ - "Returns immediately - good for conditionals.", - "Use expect().toBeVisible() for assertions.", - "Good for flow control.", + "Checks that the element is visible on the page.", + "Automatically waits for the condition to be met.", + "Throws an error if the element is not visible.", ], }, { - signature: "await page.title()", - description: "Gets the page title.", + signature: "await expect(locator).toHaveText(expectedText)", + description: "Asserts that the element's text content matches the expected text.", example: `const page = getCurrentPage(); -if (page) { - const title = await page.title(); - await expect(title).toBe('Expected Title'); -}`, + if (page) { + await expect(page.getByTestId('username')).toHaveText('John Doe'); + }`, + guidelines: [ + "Compares the element's text content with the expected text.", + "Supports partial matches and regular expressions.", + "Waits until the condition is met or times out.", + ], + }, + { + signature: "await expect(page).toHaveURL(expectedURL)", + description: "Asserts that the page's URL matches the expected URL.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page).toHaveURL('https://www.example.com/dashboard'); + }`, + guidelines: [ + "Waits for the URL to match the expected value.", + "Can use regular expressions or glob patterns for matching.", + "Useful for verifying navigation.", + ], + }, + { + signature: "await expect(locator).toHaveValue(expectedValue)", + description: "Asserts that an input element has the expected value.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByPlaceholder('Email')).toHaveValue('user@example.com'); + }`, + guidelines: [ + "Checks the 'value' property of input elements.", + "Automatically waits for the condition.", + "Useful for validating user input.", + ], + }, + { + signature: "await expect(locator).toBeEnabled()", + description: "Asserts that the element is enabled.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByRole('button', { name: 'Submit' })).toBeEnabled(); + }`, + guidelines: [ + "Checks that the element can be interacted with.", + "Waits until the element is enabled.", + "Throws an error if the element remains disabled.", + ], + }, + { + signature: "await expect(locator).toBeDisabled()", + description: "Asserts that the element is disabled.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByRole('button', { name: 'Submit' })).toBeDisabled(); + }`, + guidelines: [ + "Checks that the element is not interactive.", + "Waits until the element is disabled.", + "Useful for ensuring correct UI state.", + ], + }, + { + signature: "await expect(locator).toBeChecked()", + description: "Asserts that a checkbox or radio button is checked.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByLabel('Accept Terms')).toBeChecked(); + }`, + guidelines: [ + "Checks that the element is checked.", + "Waits for the condition to be met.", + "Throws an error if the element is not checked.", + ], + }, + { + signature: "await expect(locator).toBeEditable()", + description: "Asserts that an input or textarea element is editable.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByPlaceholder('Enter your name')).toBeEditable(); + }`, guidelines: [ - "Returns current page title.", - "Use with expect for assertions.", - "Auto-waits for title to be available.", + "Checks that the element can be edited.", + "Waits until the element is editable.", + "Useful before attempting to fill an input.", + ], + }, + { + signature: "await expect(locator).toHaveAttribute(name, value)", + description: "Asserts that the element has the specified attribute value.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByTestId('profile-link')).toHaveAttribute('href', '/profile'); + }`, + guidelines: [ + "Checks the value of a specified attribute.", + "Supports regular expressions and partial matching.", + "Waits for the condition to be met.", + ], + }, + { + signature: "await expect(locator).toHaveClass(expectedClass)", + description: "Asserts that the element has the specified CSS class.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByRole('button', { name: 'Submit' })).toHaveClass('btn-primary'); + }`, + guidelines: [ + "Checks the 'class' attribute of the element.", + "Supports multiple classes and partial matches.", + "Waits for the condition to be met.", + ], + }, + { + signature: "await expect(locator).toContainText(expectedText)", + description: "Asserts that the element's text contains the expected text.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByTestId('notification')).toContainText('Success'); + }`, + guidelines: [ + "Checks that the element's text includes the expected substring.", + "Supports regular expressions and partial matches.", + "Waits for the condition to be met.", + ], + }, + { + signature: "await expect(locator).toHaveJSProperty(name, value)", + description: "Asserts that the element has the specified JavaScript property.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByTestId('toggle')).toHaveJSProperty('checked', true); + }`, + guidelines: [ + "Checks the value of a JavaScript property on the element.", + "Useful for properties not exposed via attributes.", + "Waits for the condition to be met.", + ], + }, + { + signature: "await expect(locator).not.toBeVisible()", + description: "Asserts that the element is not visible.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByText('Loading...')).not.toBeVisible(); + }`, + guidelines: [ + "Checks that the element is hidden or does not exist.", + "Waits for the condition to be met.", + "Useful for ensuring elements are not present before proceeding.", + ], + }, + { + signature: "await expect(page).toHaveTitle(expectedTitle)", + description: "Asserts that the page's title matches the expected title.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page).toHaveTitle('Dashboard - MyApp'); + }`, + guidelines: [ + "Waits for the page's title to match the expected value.", + "Supports regular expressions and partial matching.", + "Useful for verifying page navigation.", + ], + }, + { + signature: "await expect(page).toHaveScreenshot([options])", + description: "Asserts that the page's screenshot matches a stored reference image.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page).toHaveScreenshot('dashboard.png'); + }`, + guidelines: [ + "Compares the current screenshot with a baseline image.", + "Can specify options like threshold and mask areas.", + "Useful for visual regression testing.", + ], + }, + { + signature: "await expect(locator).toHaveCount(expectedCount)", + description: "Asserts that the locator resolves to a specific number of elements.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.locator('.todo-item')).toHaveCount(3); + }`, + guidelines: [ + "Checks the number of elements matching the locator.", + "Waits for the condition to be met.", + "Useful for validating dynamic lists.", + ], + }, + { + signature: "await expect(locator).toBeEmpty()", + description: "Asserts that the element is empty.", + example: `const page = getCurrentPage(); + if (page) { + await expect(page.getByTestId('search-results')).toBeEmpty(); + }`, + guidelines: [ + "Checks that the element has no text content.", + "Waits for the condition to be met.", + "Useful for asserting initial states.", ], }, ], - }, + } ], }; } diff --git a/src/integration tests/index.test.ts b/src/integration tests/index.test.ts index 9119d7f..2b51fe1 100644 --- a/src/integration tests/index.test.ts +++ b/src/integration tests/index.test.ts @@ -500,6 +500,7 @@ describe("Copilot Integration Tests", () => { goal: goal, steps: [ { + screenName: "Login Screen", // Added screenName plan: { thoughts: "First step thoughts", action: "Tap on login button", @@ -561,6 +562,7 @@ describe("Copilot Integration Tests", () => { }); }); + describe("Cache Modes", () => { beforeEach(() => { mockPromptHandler.runPrompt.mockResolvedValue("// No operation"); diff --git a/src/types/framework.ts b/src/types/framework.ts index 43e9f84..36250e5 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -32,6 +32,8 @@ export type TestingFrameworkAPICatalog = { context: any; /** Available API method categories */ categories: TestingFrameworkAPICatalogCategory[]; + /** List of restrictions and guidlines of wrong actions */ + restrictions?: string[]; }; /** diff --git a/src/types/pilot.ts b/src/types/pilot.ts index 88d8c72..30ee23b 100644 --- a/src/types/pilot.ts +++ b/src/types/pilot.ts @@ -22,6 +22,8 @@ export type PilotReview = { * Single pilot step execution report. */ export type PilotStepReport = { + /** Screen name of the current view */ + screenName: string; /** Action plan and reasoning */ plan: PilotStepPlan; /** Optional reviews */ @@ -74,6 +76,8 @@ export type PilotReviewSection = { * Previous pilot step record. */ export type PilotPreviousStep = { + /** Screen name */ + screenName: string; /** Step description */ step: string; /** Optional reviews */ diff --git a/src/utils/PilotPromptCreator.test.ts b/src/utils/PilotPromptCreator.test.ts index a72fefd..f5de9c3 100644 --- a/src/utils/PilotPromptCreator.test.ts +++ b/src/utils/PilotPromptCreator.test.ts @@ -1,5 +1,5 @@ import { PilotPromptCreator } from "./PilotPromptCreator"; -import { PreviousStep } from "@/types"; +import { PilotPreviousStep } from "@/types"; describe("PilotPromptCreator", () => { let promptCreator: PilotPromptCreator; @@ -18,16 +18,15 @@ describe("PilotPromptCreator", () => { it("should include previous intents in the context", () => { const intent = "tap button"; - const previousSteps: PreviousStep[] = [ + const previousSteps: PilotPreviousStep[] = [ { + screenName: "default 1", step: "navigate to login screen", - code: 'await element(by.id("login")).tap();', - result: undefined, + }, { + screenName: "default 2", step: "enter username", - code: 'await element(by.id("username")).typeText("john_doe");', - result: undefined, }, ]; diff --git a/src/utils/PilotPromptCreator.ts b/src/utils/PilotPromptCreator.ts index 7057f66..ed3a55e 100644 --- a/src/utils/PilotPromptCreator.ts +++ b/src/utils/PilotPromptCreator.ts @@ -161,7 +161,10 @@ export class PilotPromptCreator { "", "#### Next Action with Thoughts:", "", - ` + ` +Registration Page. + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -198,7 +201,10 @@ The 'Submit' button (ID: btn_submit) lacks essential accessibility features. "", "#### Example of Success:", "", - ` + ` +Goal Page Name. + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -233,7 +239,10 @@ An overall accessibility review summary based on the previous steps' reviews. "", "#### Additional Example:", "", - ` + ` +User Profile Screen. + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. @@ -283,6 +292,7 @@ The 'Profile' icon (ID: icon_profile) has accessibility issues that could affect `Consider the elements present in the view hierarchy${isSnapshotImageAttached ? " and the snapshot image" : ""} to determine possible next actions.`, "Determine the optimal next action the user should take to move closer to their goal.", "Ensure the action is directly related to available elements in the view hierarchy.", + "Make sure to create a unique screen name enclosed with blocks according to the snapshot and view", "Generate a one-line string that precisely describes this next action, enclosed within `` tags.", "Provide a detailed description of your thought process enclosed within `` tags.", "If the goal is achieved, add a `` block inside the `` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections.", diff --git a/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap b/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap index aeb65b4..3b404c9 100644 --- a/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap +++ b/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap @@ -43,12 +43,13 @@ Your tasks are as follows: 2. Consider the elements present in the view hierarchy and the snapshot image to determine possible next actions. 3. Determine the optimal next action the user should take to move closer to their goal. 4. Ensure the action is directly related to available elements in the view hierarchy. -5. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. -6. Provide a detailed description of your thought process enclosed within \`\` tags. -7. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. -8. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. -9. Ensure each section is clearly labeled and formatted as shown in the examples. -10. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. +5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view +6. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. +7. Provide a detailed description of your thought process enclosed within \`\` tags. +8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. +9. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. +10. Ensure each section is clearly labeled and formatted as shown in the examples. +11. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. ### Verify the Prompt @@ -59,7 +60,10 @@ If you encounter any issues or have questions, please throw an informative error #### Next Action with Thoughts: - + +Registration Page. + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -95,7 +99,10 @@ The 'Submit' button (ID: btn_submit) lacks essential accessibility features. #### Example of Success: - + +Goal Page Name. + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -129,7 +136,10 @@ An overall accessibility review summary based on the previous steps' reviews. #### Additional Example: - + +User Profile Screen. + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. @@ -212,12 +222,13 @@ Your tasks are as follows: 2. Consider the elements present in the view hierarchy to determine possible next actions. 3. Determine the optimal next action the user should take to move closer to their goal. 4. Ensure the action is directly related to available elements in the view hierarchy. -5. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. -6. Provide a detailed description of your thought process enclosed within \`\` tags. -7. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. -8. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. -9. Ensure each section is clearly labeled and formatted as shown in the examples. -10. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. +5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view +6. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. +7. Provide a detailed description of your thought process enclosed within \`\` tags. +8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. +9. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. +10. Ensure each section is clearly labeled and formatted as shown in the examples. +11. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. ### Verify the Prompt @@ -228,7 +239,10 @@ If you encounter any issues or have questions, please throw an informative error #### Next Action with Thoughts: - + +Registration Page. + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -264,7 +278,10 @@ The 'Submit' button (ID: btn_submit) lacks essential accessibility features. #### Example of Success: - + +Goal Page Name. + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -298,7 +315,10 @@ An overall accessibility review summary based on the previous steps' reviews. #### Additional Example: - + +User Profile Screen. + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. @@ -390,12 +410,13 @@ Your tasks are as follows: 2. Consider the elements present in the view hierarchy to determine possible next actions. 3. Determine the optimal next action the user should take to move closer to their goal. 4. Ensure the action is directly related to available elements in the view hierarchy. -5. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. -6. Provide a detailed description of your thought process enclosed within \`\` tags. -7. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. -8. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. -9. Ensure each section is clearly labeled and formatted as shown in the examples. -10. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. +5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view +6. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. +7. Provide a detailed description of your thought process enclosed within \`\` tags. +8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. +9. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. +10. Ensure each section is clearly labeled and formatted as shown in the examples. +11. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. ### Verify the Prompt @@ -406,7 +427,10 @@ If you encounter any issues or have questions, please throw an informative error #### Next Action with Thoughts: - + +Registration Page. + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -442,7 +466,10 @@ The 'Submit' button (ID: btn_submit) lacks essential accessibility features. #### Example of Success: - + +Goal Page Name. + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -476,7 +503,10 @@ An overall accessibility review summary based on the previous steps' reviews. #### Additional Example: - + +User Profile Screen. + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. diff --git a/src/utils/extractOutputs.ts b/src/utils/extractOutputs.ts index 65b6aae..8624d90 100644 --- a/src/utils/extractOutputs.ts +++ b/src/utils/extractOutputs.ts @@ -12,6 +12,7 @@ export const OUTPUTS_MAPPINGS: Record = { score: { tag: "SCORE", isRequired: false }, }, PILOT_STEP: { + screenName: { tag: "SCREENNAME", isRequired: true }, thoughts: { tag: "THOUGHTS", isRequired: true }, action: { tag: "ACTION", isRequired: true }, ux: { tag: "UX", isRequired: false }, diff --git a/src/utils/logger/index.ts b/src/utils/logger/index.ts index a126d26..a946045 100644 --- a/src/utils/logger/index.ts +++ b/src/utils/logger/index.ts @@ -12,6 +12,7 @@ import { LoggerMessageColor, LoggerOperationResultType, } from "@/types/logger"; +import * as fs from "fs"; class Logger { private static instance: Logger; @@ -26,6 +27,7 @@ class Logger { error: "red", debug: "gray", }; + private logs: string[] = []; private constructor() { this.logger = createLogger({ @@ -67,13 +69,41 @@ class Logger { .join(" "); } + private formatMessage(...components: LoggerMessageComponent[]): string { + return components + .map((component) => { + if (typeof component === "string") { + return component; + } + + let message = component.message; + + if (component.isBold) { + message = `**${message}**`; + } + + return message; + }) + .join(" "); + } + + private formatTimestamp(date: Date): string { + const pad = (n: number) => (n < 10 ? "0" + n : n.toString()); + const year = date.getFullYear(); + const month = pad(date.getMonth() + 1); + const day = pad(date.getDate()); + const hours = pad(date.getHours()); + const minutes = pad(date.getMinutes()); + const seconds = pad(date.getSeconds()); + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; + } + private log( level: (typeof this.logLevels)[number], ...components: LoggerMessageComponent[] ): void { const newComponents = components.map((component) => { if (typeof component === "string") { - // Overriding the component to include the specified color return { message: component, isBold: false, @@ -84,7 +114,15 @@ class Logger { return component; }); + this.logger[level](this.colorizeMessage(...newComponents)); + + const timestamp = this.formatTimestamp(new Date()); + const levelUpper = level.toUpperCase(); + const messageText = this.formatMessage(...components); + const logEntry = `[${timestamp}] ${levelUpper}: ${messageText}`; + + this.logs.push(logEntry); } public info(...components: LoggerMessageComponent[]): void { @@ -103,7 +141,9 @@ class Logger { this.log("debug", ...components); } - public startSpinner(...components: LoggerMessageComponent[]): LoggerSpinner { + public startSpinner( + ...components: LoggerMessageComponent[] + ): LoggerSpinner { const spinner = ora(this.colorizeMessage(...components)).start(); const stop = ( @@ -113,7 +153,10 @@ class Logger { spinner.prefixText = ""; const message = this.colorizeMessage(...components); - const spinnerActions: Record void> = { + const spinnerActions: Record< + LoggerOperationResultType, + () => ora.Ora + > = { success: () => spinner.succeed(message), failure: () => spinner.fail(message), warn: () => spinner.warn(message), @@ -129,6 +172,19 @@ class Logger { return { update, stop }; } + + public writeLogsToFile(filename: string): void { + try { + fs.writeFileSync(filename, this.logs.join("\n"), "utf8"); + this.info(`Logs have been written to ${filename}`); + } catch (err) { + this.error("Failed to write logs to file:", { + message: `${(err as Error).message}`, + isBold: false, + color: "red", + }); + } + } } -export default Logger.getInstance(); +export default Logger.getInstance(); \ No newline at end of file From fcfd94735ea50f8a971e94708bb7fc27851442b9 Mon Sep 17 00:00:00 2001 From: Tzviel Solodar Date: Sun, 26 Jan 2025 18:50:52 +0200 Subject: [PATCH 2/4] wip: add screen name to prompt creator --- src/utils/PilotPromptCreator.ts | 1 + src/utils/__snapshots__/PilotPromptCreator.test.ts.snap | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/utils/PilotPromptCreator.ts b/src/utils/PilotPromptCreator.ts index ed3a55e..67b4123 100644 --- a/src/utils/PilotPromptCreator.ts +++ b/src/utils/PilotPromptCreator.ts @@ -83,6 +83,7 @@ export class PilotPromptCreator { .map((previousStep, index) => { const stepDetails = [ `#### Step ${index + 1}`, + `- Screen Name : "${previousStep.screenName}"`, `- Intent: "${previousStep.step}"`, ]; diff --git a/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap b/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap index 3b404c9..3e322e0 100644 --- a/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap +++ b/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap @@ -388,9 +388,11 @@ No snapshot image is attached for this intent. ### Previous Steps #### Step 1 +- Screen Name : "default 1" - Intent: "navigate to login screen" #### Step 2 +- Screen Name : "default 2" - Intent: "enter username" From 20d69458c36b436c6ca4602613915d94da557df5 Mon Sep 17 00:00:00 2001 From: Tzviel Solodar Date: Mon, 27 Jan 2025 13:26:37 +0200 Subject: [PATCH 3/4] fixes after review + add i18n to pilot review options --- src/Copilot.test.ts | 12 +- src/actions/PilotPerformer.test.ts | 83 +++++- src/actions/PilotPerformer.ts | 29 +- src/integration tests/index.test.ts | 2 +- src/types/pilot.ts | 8 +- src/utils/PilotPromptCreator.test.ts | 4 +- src/utils/PilotPromptCreator.ts | 101 +++++-- .../PilotPromptCreator.test.ts.snap | 273 ++++++++++++------ src/utils/extractOutputs.ts | 3 +- src/utils/logger/index.ts | 20 +- 10 files changed, 374 insertions(+), 161 deletions(-) diff --git a/src/Copilot.test.ts b/src/Copilot.test.ts index a72fa78..321fa9b 100644 --- a/src/Copilot.test.ts +++ b/src/Copilot.test.ts @@ -330,7 +330,7 @@ describe("Copilot", () => { goal, steps: [ { - screenName: "Screen 1", + screenDescription: "Screen 1", plan: { thoughts: "Step 1 thoughts", action: "Tap on GREAT button", @@ -339,7 +339,7 @@ describe("Copilot", () => { goalAchieved: false, }, { - screenName: "Screen 2", + screenDescription: "Screen 2", plan: { thoughts: "Completed successfully", action: "success", @@ -365,7 +365,7 @@ describe("Copilot", () => { const goal = "Test the login flow"; const pilotOutputStep1: PilotStepReport = { - screenName: "Login Screen", + screenDescription: "Login Screen", plan: { thoughts: "Step 1 thoughts", action: "Tap on Login button", @@ -386,7 +386,7 @@ describe("Copilot", () => { }; const pilotOutputSuccess: PilotStepReport = { - screenName: "Home Screen", + screenDescription: "Home Screen", plan: { thoughts: "Completed successfully", action: "success", @@ -414,7 +414,7 @@ describe("Copilot", () => { goal: goal, steps: [ { - screenName: pilotOutputStep1.screenName, + screenDescription: pilotOutputStep1.screenDescription, plan: pilotOutputStep1.plan, code: "code executed", review: pilotOutputStep1.review, @@ -432,7 +432,7 @@ describe("Copilot", () => { goal: goal, steps: [ { - screenName: pilotOutputStep1.screenName, + screenDescription: pilotOutputStep1.screenDescription, plan: pilotOutputStep1.plan, code: "code executed", review: pilotOutputStep1.review, diff --git a/src/actions/PilotPerformer.test.ts b/src/actions/PilotPerformer.test.ts index 482ac5a..522ac17 100644 --- a/src/actions/PilotPerformer.test.ts +++ b/src/actions/PilotPerformer.test.ts @@ -14,11 +14,11 @@ const GOAL = "tap button"; const VIEW_HIERARCHY = ""; const GENERATED_PROMPT = "generated prompt"; -// Updated PROMPT_RESULT to include screenName, UX, and Accessibility sections +// Updated PROMPT_RESULT to include screenDescription, UX, Accessibility, and Internationalization sections const PROMPT_RESULT = ` - + default name - + I think this is great @@ -48,7 +48,19 @@ The review of accessibility 8/10 -`; + + + +The review of i18n + + +- i18n finding one +- i18n finding two + + +6/10 + +`; const SNAPSHOT_DATA = "snapshot_data"; @@ -128,7 +140,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { - screenName: "default name", + screenDescription: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -144,6 +156,11 @@ describe("PilotPerformer", () => { findings: ["ACC finding one", "ACC finding two"], score: "8/10", }, + i18n: { + summary: "The review of i18n", + findings: ["i18n finding one", "i18n finding two"], + score: "6/10", + }, }, goalAchieved: false, }; @@ -171,7 +188,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { - screenName: "default name", + screenDescription: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -187,6 +204,11 @@ describe("PilotPerformer", () => { findings: ["ACC finding one", "ACC finding two"], score: "8/10", }, + i18n: { + summary: "The review of i18n", + findings: ["i18n finding one", "i18n finding two"], + score: "6/10", + }, }, goalAchieved: false, }; @@ -214,7 +236,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { - screenName: "default name", + screenDescription: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -230,6 +252,11 @@ describe("PilotPerformer", () => { findings: ["ACC finding one", "ACC finding two"], score: "8/10", }, + i18n: { + summary: "The review of i18n", + findings: ["i18n finding one", "i18n finding two"], + score: "6/10", + }, }, goalAchieved: false, }; @@ -251,8 +278,25 @@ describe("PilotPerformer", () => { const intent = "current intent"; const previousIntents: PilotPreviousStep[] = [ { - screenName: "default", + screenDescription: "default", step: "previous intent", + review: { + ux: { + summary: "Previous UX summary", + findings: ["Previous UX finding"], + score: "6/10", + }, + a11y: { + summary: "Previous Accessibility summary", + findings: ["Previous ACC finding"], + score: "7/10", + }, + i18n: { + summary: "Previous i18n summary", + findings: ["Previous i18n finding"], + score: "5/10", + }, + }, }, ]; @@ -265,7 +309,7 @@ describe("PilotPerformer", () => { ); const expectedResult = { - screenName: "default name", + screenDescription: "default name", plan: { thoughts: "I think this is great", action: "Tap on GREAT button", @@ -281,6 +325,11 @@ describe("PilotPerformer", () => { findings: ["ACC finding one", "ACC finding two"], score: "8/10", }, + i18n: { + summary: "The review of i18n", + findings: ["i18n finding one", "i18n finding two"], + score: "6/10", + }, }, goalAchieved: false, }; @@ -301,7 +350,7 @@ describe("PilotPerformer", () => { describe("perform", () => { it("should perform multiple steps until success is returned", async () => { const pilotOutputStep1: PilotStepReport = { - screenName: "Screen 1", + screenDescription: "Screen 1", plan: { thoughts: "Step 1 thoughts", action: "Tap on GREAT button", @@ -317,12 +366,17 @@ describe("PilotPerformer", () => { findings: [], score: "8/10", }, + i18n: { + summary: "i18n review for step 1", + findings: [], + score: "6/10", + }, }, goalAchieved: false, }; const pilotOutputSuccess: PilotStepReport = { - screenName: "Screen 2", + screenDescription: "Screen 2", plan: { thoughts: "Completed successfully", action: "success", @@ -338,6 +392,11 @@ describe("PilotPerformer", () => { findings: [], score: "9/10", }, + i18n: { + summary: "Final i18n review", + findings: [], + score: "8/10", + }, }, goalAchieved: true, summary: "all was good", @@ -373,7 +432,7 @@ describe("PilotPerformer", () => { goal: GOAL, steps: [ { - screenName: "Screen 1", + screenDescription: "Screen 1", plan: pilotOutputStep1.plan, code: "code executed", review: pilotOutputStep1.review, diff --git a/src/actions/PilotPerformer.ts b/src/actions/PilotPerformer.ts index e6a3275..f372412 100644 --- a/src/actions/PilotPerformer.ts +++ b/src/actions/PilotPerformer.ts @@ -40,7 +40,7 @@ export class PilotPerformer { }; } - private logReviewSection(review: PilotReviewSection, type: "ux" | "a11y") { + private logReviewSection(review: PilotReviewSection, type: "ux" | "a11y" | "i18n") { const config: { [key: string]: { emoji: string; @@ -58,6 +58,11 @@ export class PilotPerformer { color: "yellowBright", findingColor: "yellow", }, + i18n: { + emoji: "🌐", + color: "cyanBright", + findingColor: "cyan", + }, }; logger.info({ @@ -97,7 +102,7 @@ export class PilotPerformer { const generatedPilotTaskDetails: string = await this.promptHandler.runPrompt(prompt, snapshot); - const { screenName, thoughts, action, ux, a11y } = extractOutputs({ + const { screenDescription, thoughts, action, ux, a11y, i18n } = extractOutputs({ text: generatedPilotTaskDetails, outputsMapper: OUTPUTS_MAPPINGS.PILOT_STEP, }); @@ -107,21 +112,23 @@ export class PilotPerformer { isBold: true, color: "whiteBright", }); - + const plan: PilotStepPlan = { action, thoughts }; const review: PilotReview = { ux: this.extractReviewOutput(ux), a11y: this.extractReviewOutput(a11y), + i18n: this.extractReviewOutput(i18n), }; - + logger.info({ - message: `Conducting review for ${screenName}\n`, + message: `Conducting review for ${screenDescription}\n`, isBold: true, color: 'whiteBright', }); review.ux && this.logReviewSection(review.ux, "ux"); review.a11y && this.logReviewSection(review.a11y, "a11y"); + review.i18n && this.logReviewSection(review.i18n, "i18n"); const goalAchieved = action === "success"; @@ -132,7 +139,7 @@ export class PilotPerformer { }).summary : undefined; - return { screenName, plan, review, goalAchieved, summary }; + return { screenDescription, plan, review, goalAchieved, summary }; } catch (error) { analysisLoggerSpinner.stop( "failure", @@ -164,7 +171,7 @@ export class PilotPerformer { for (let step = 0; step < maxSteps; step++) { const screenCapture: ScreenCapturerResult = await this.screenCapturer.capture(); - const { screenName, plan, review, goalAchieved, summary } = + const { screenDescription, plan, review, goalAchieved, summary } = await this.analyseScreenAndCreateCopilotStep( goal, previousSteps, @@ -180,17 +187,17 @@ export class PilotPerformer { logger.writeLogsToFile(`pilot_logs_${Date.now()}`); return { goal, summary, steps: [...report.steps], review }; } - + const { code, result } = await this.copilotStepPerformer.perform( plan.action, [...copilotSteps], screenCapture, ); copilotSteps = [...copilotSteps, { step: plan.action, code, result }]; - previousSteps = [...previousSteps, { screenName, step: plan.action, review }]; + previousSteps = [...previousSteps, { screenDescription, step: plan.action, review }]; const stepReport: PilotStepReport = { - screenName, + screenDescription, plan, review, code, @@ -206,4 +213,4 @@ export class PilotPerformer { logger.writeLogsToFile(`pilot_logs_${Date.now()}`); return report; } -} +} \ No newline at end of file diff --git a/src/integration tests/index.test.ts b/src/integration tests/index.test.ts index 2b51fe1..aa4a460 100644 --- a/src/integration tests/index.test.ts +++ b/src/integration tests/index.test.ts @@ -500,7 +500,7 @@ describe("Copilot Integration Tests", () => { goal: goal, steps: [ { - screenName: "Login Screen", // Added screenName + screenDescription: "Login Screen", // Added screenDescription plan: { thoughts: "First step thoughts", action: "Tap on login button", diff --git a/src/types/pilot.ts b/src/types/pilot.ts index 30ee23b..fa34ed7 100644 --- a/src/types/pilot.ts +++ b/src/types/pilot.ts @@ -11,7 +11,7 @@ export type PreviousStep = { }; /** Review section types */ -export type PilotReviewSectionType = "ux" | "a11y"; +export type PilotReviewSectionType = "ux" | "a11y"| "i18n"; /** Complete pilot review */ export type PilotReview = { @@ -23,7 +23,7 @@ export type PilotReview = { */ export type PilotStepReport = { /** Screen name of the current view */ - screenName: string; + screenDescription: string; /** Action plan and reasoning */ plan: PilotStepPlan; /** Optional reviews */ @@ -76,8 +76,8 @@ export type PilotReviewSection = { * Previous pilot step record. */ export type PilotPreviousStep = { - /** Screen name */ - screenName: string; + /** Screen description */ + screenDescription: string; /** Step description */ step: string; /** Optional reviews */ diff --git a/src/utils/PilotPromptCreator.test.ts b/src/utils/PilotPromptCreator.test.ts index f5de9c3..5b94166 100644 --- a/src/utils/PilotPromptCreator.test.ts +++ b/src/utils/PilotPromptCreator.test.ts @@ -20,12 +20,12 @@ describe("PilotPromptCreator", () => { const intent = "tap button"; const previousSteps: PilotPreviousStep[] = [ { - screenName: "default 1", + screenDescription: "default 1", step: "navigate to login screen", }, { - screenName: "default 2", + screenDescription: "default 2", step: "enter username", }, ]; diff --git a/src/utils/PilotPromptCreator.ts b/src/utils/PilotPromptCreator.ts index 67b4123..ee4bb0d 100644 --- a/src/utils/PilotPromptCreator.ts +++ b/src/utils/PilotPromptCreator.ts @@ -25,7 +25,7 @@ export class PilotPromptCreator { private createBasePrompt(): string[] { return [ - "# Next Step Generation and UX/Accessibility Reporting", + "# Next Step Generation and UX/Accessibility/Internationalization Reporting", "", "You are an AI assistant tasked with:", "", @@ -33,7 +33,7 @@ export class PilotPromptCreator { ` Please generate a one-line string that precisely describes the next action the user should take to move closer to their goal,`, ` and another string (which can be greater than one line) which describes your thoughts while creating the step.`, ` If you think that the goal has been reached return a one word action 'success'`, - "2. Creating comprehensive UX and accessibility reports that include a review, findings, and a score.", + "2. Creating comprehensive UX, Accessibility, and Internationalization reports that include a review, findings, and a score.", "", "Please adhere to the instructions below to provide detailed and helpful responses.", "", @@ -83,7 +83,7 @@ export class PilotPromptCreator { .map((previousStep, index) => { const stepDetails = [ `#### Step ${index + 1}`, - `- Screen Name : "${previousStep.screenName}"`, + `- Screen Name : "${previousStep.screenDescription}"`, `- Intent: "${previousStep.step}"`, ]; @@ -127,6 +127,8 @@ export class PilotPromptCreator { return "UX"; case "a11y": return "Accessibility"; + case "i18n": + return "Internationalization"; default: throw new Error(`Invalid review section: ${sectionType}`); } @@ -162,10 +164,10 @@ export class PilotPromptCreator { "", "#### Next Action with Thoughts:", "", - ` + ` Registration Page. - - + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -177,9 +179,9 @@ Tap on the 'Submit' button, which has the ID 'btn_submit' and is located at the The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potentially hindering the registration process. - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. - It uses a color that blends with the background - Use a contrasting color to make the button stand out. - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. + - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. + - It uses a color that blends with the background - Use a contrasting color to make the button stand out. + - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. 6/10 @@ -190,22 +192,35 @@ The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potent The 'Submit' button (ID: btn_submit) lacks essential accessibility features. - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. + - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. + - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. + - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. 5/10 -`, + + + +The 'Submit' button may not be properly localized for all supported languages. + + + - The button text is hard-coded in English - Use localization files to support multiple languages. + - Layout may break for languages with longer text strings - Ensure dynamic resizing or text wrapping is implemented. + - Missing support for right-to-left languages - Adjust layout to support RTL languages where necessary. + + +4/10 + +`, "", "#### Example of Success:", "", - ` + ` Goal Page Name. - - + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -219,10 +234,10 @@ success An overall UX review summary based on the previous steps' reviews. -Summary of UX findings from previous steps. + - Summary of UX findings from previous steps. -6/10 - This is an overall score for the entire flow. +7/10 - This is an overall score for the entire flow. @@ -230,20 +245,31 @@ Summary of UX findings from previous steps. An overall accessibility review summary based on the previous steps' reviews. - Summary of accessibility findings from previous steps. + - Summary of accessibility findings from previous steps. + + +6/10 - This is an overall score for the entire flow. + + + + +An overall internationalization review summary based on the previous steps' reviews. + + + - Summary of internationalization findings from previous steps. 5/10 - This is an overall score for the entire flow. -`, +`, "", "#### Additional Example:", "", - ` + ` User Profile Screen. - - + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. @@ -254,8 +280,8 @@ Select the 'Profile' icon (ID: icon_profile) The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all users. - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. + - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. + - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. 5/10 @@ -266,13 +292,26 @@ The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all The 'Profile' icon (ID: icon_profile) has accessibility issues that could affect users with disabilities. - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. + - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. + - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. 4/10 -`, + + + +The 'Profile' icon may not be properly adapted for different locales. + + + - Iconography may not be universally recognized - Consider using culturally neutral icons. + - No localization for the tooltip text - Ensure tooltips and labels are localized. + - Date and time formats on the profile screen may not match user locale - Use locale-aware date and time formats. + + +6/10 + +`, "", "#### Example of Throwing an Informative Error:", @@ -293,14 +332,14 @@ The 'Profile' icon (ID: icon_profile) has accessibility issues that could affect `Consider the elements present in the view hierarchy${isSnapshotImageAttached ? " and the snapshot image" : ""} to determine possible next actions.`, "Determine the optimal next action the user should take to move closer to their goal.", "Ensure the action is directly related to available elements in the view hierarchy.", - "Make sure to create a unique screen name enclosed with blocks according to the snapshot and view", + "Make sure to create a unique screen name enclosed with blocks according to the snapshot and view.", "Generate a one-line string that precisely describes this next action, enclosed within `` tags.", "Provide a detailed description of your thought process enclosed within `` tags.", - "If the goal is achieved, add a `` block inside the `` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections.", + "If the goal is achieved, add a `` block inside the `` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX, Accessibility, and Internationalization reviews with total scores, given all the screens seen in previous steps, inside the respective review sections.", "For each applicable review section (`UX`, `Accessibility`, `Internationalization`), create a comprehensive report enclosed within corresponding tags (e.g., ``, ``, ``), including a summary, findings, and a score out of 10.", "Ensure each section is clearly labeled and formatted as shown in the examples.", "If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence.", ]; return steps; } -} +} \ No newline at end of file diff --git a/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap b/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap index 3e322e0..8debba4 100644 --- a/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap +++ b/src/utils/__snapshots__/PilotPromptCreator.test.ts.snap @@ -1,7 +1,7 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`PilotPromptCreator should create a prompt for an intent correctly 1`] = ` -"# Next Step Generation and UX/Accessibility Reporting +"# Next Step Generation and UX/Accessibility/Internationalization Reporting You are an AI assistant tasked with: @@ -9,7 +9,7 @@ You are an AI assistant tasked with: Please generate a one-line string that precisely describes the next action the user should take to move closer to their goal, and another string (which can be greater than one line) which describes your thoughts while creating the step. If you think that the goal has been reached return a one word action 'success' -2. Creating comprehensive UX and accessibility reports that include a review, findings, and a score. +2. Creating comprehensive UX, Accessibility, and Internationalization reports that include a review, findings, and a score. Please adhere to the instructions below to provide detailed and helpful responses. @@ -43,10 +43,10 @@ Your tasks are as follows: 2. Consider the elements present in the view hierarchy and the snapshot image to determine possible next actions. 3. Determine the optimal next action the user should take to move closer to their goal. 4. Ensure the action is directly related to available elements in the view hierarchy. -5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view +5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view. 6. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. 7. Provide a detailed description of your thought process enclosed within \`\` tags. -8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. +8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX, Accessibility, and Internationalization reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. 9. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. 10. Ensure each section is clearly labeled and formatted as shown in the examples. 11. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. @@ -60,10 +60,10 @@ If you encounter any issues or have questions, please throw an informative error #### Next Action with Thoughts: - + Registration Page. - - + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -75,9 +75,9 @@ Tap on the 'Submit' button, which has the ID 'btn_submit' and is located at the The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potentially hindering the registration process. - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. - It uses a color that blends with the background - Use a contrasting color to make the button stand out. - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. + - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. + - It uses a color that blends with the background - Use a contrasting color to make the button stand out. + - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. 6/10 @@ -88,21 +88,34 @@ The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potent The 'Submit' button (ID: btn_submit) lacks essential accessibility features. - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. + - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. + - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. + - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. 5/10 + + +The 'Submit' button may not be properly localized for all supported languages. + + + - The button text is hard-coded in English - Use localization files to support multiple languages. + - Layout may break for languages with longer text strings - Ensure dynamic resizing or text wrapping is implemented. + - Missing support for right-to-left languages - Adjust layout to support RTL languages where necessary. + + +4/10 + + #### Example of Success: - + Goal Page Name. - - + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -116,10 +129,10 @@ success An overall UX review summary based on the previous steps' reviews. -Summary of UX findings from previous steps. + - Summary of UX findings from previous steps. -6/10 - This is an overall score for the entire flow. +7/10 - This is an overall score for the entire flow. @@ -127,19 +140,30 @@ Summary of UX findings from previous steps. An overall accessibility review summary based on the previous steps' reviews. - Summary of accessibility findings from previous steps. + - Summary of accessibility findings from previous steps. -5/10 - This is an overall score for the entire flow. +6/10 - This is an overall score for the entire flow. + + +An overall internationalization review summary based on the previous steps' reviews. + + + - Summary of internationalization findings from previous steps. + + +5/10 - This is an overall score for the entire flow. + + #### Additional Example: - + User Profile Screen. - - + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. @@ -150,8 +174,8 @@ Select the 'Profile' icon (ID: icon_profile) The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all users. - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. + - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. + - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. 5/10 @@ -162,13 +186,26 @@ The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all The 'Profile' icon (ID: icon_profile) has accessibility issues that could affect users with disabilities. - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. + - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. + - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. 4/10 + + +The 'Profile' icon may not be properly adapted for different locales. + + + - Iconography may not be universally recognized - Consider using culturally neutral icons. + - No localization for the tooltip text - Ensure tooltips and labels are localized. + - Date and time formats on the profile screen may not match user locale - Use locale-aware date and time formats. + + +6/10 + + #### Example of Throwing an Informative Error: @@ -180,7 +217,7 @@ Please provide your response below:" `; exports[`PilotPromptCreator should handle when no snapshot image is attached 1`] = ` -"# Next Step Generation and UX/Accessibility Reporting +"# Next Step Generation and UX/Accessibility/Internationalization Reporting You are an AI assistant tasked with: @@ -188,7 +225,7 @@ You are an AI assistant tasked with: Please generate a one-line string that precisely describes the next action the user should take to move closer to their goal, and another string (which can be greater than one line) which describes your thoughts while creating the step. If you think that the goal has been reached return a one word action 'success' -2. Creating comprehensive UX and accessibility reports that include a review, findings, and a score. +2. Creating comprehensive UX, Accessibility, and Internationalization reports that include a review, findings, and a score. Please adhere to the instructions below to provide detailed and helpful responses. @@ -222,10 +259,10 @@ Your tasks are as follows: 2. Consider the elements present in the view hierarchy to determine possible next actions. 3. Determine the optimal next action the user should take to move closer to their goal. 4. Ensure the action is directly related to available elements in the view hierarchy. -5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view +5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view. 6. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. 7. Provide a detailed description of your thought process enclosed within \`\` tags. -8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. +8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX, Accessibility, and Internationalization reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. 9. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. 10. Ensure each section is clearly labeled and formatted as shown in the examples. 11. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. @@ -239,10 +276,10 @@ If you encounter any issues or have questions, please throw an informative error #### Next Action with Thoughts: - + Registration Page. - - + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -254,9 +291,9 @@ Tap on the 'Submit' button, which has the ID 'btn_submit' and is located at the The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potentially hindering the registration process. - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. - It uses a color that blends with the background - Use a contrasting color to make the button stand out. - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. + - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. + - It uses a color that blends with the background - Use a contrasting color to make the button stand out. + - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. 6/10 @@ -267,21 +304,34 @@ The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potent The 'Submit' button (ID: btn_submit) lacks essential accessibility features. - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. + - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. + - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. + - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. 5/10 + + +The 'Submit' button may not be properly localized for all supported languages. + + + - The button text is hard-coded in English - Use localization files to support multiple languages. + - Layout may break for languages with longer text strings - Ensure dynamic resizing or text wrapping is implemented. + - Missing support for right-to-left languages - Adjust layout to support RTL languages where necessary. + + +4/10 + + #### Example of Success: - + Goal Page Name. - - + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -295,10 +345,10 @@ success An overall UX review summary based on the previous steps' reviews. -Summary of UX findings from previous steps. + - Summary of UX findings from previous steps. -6/10 - This is an overall score for the entire flow. +7/10 - This is an overall score for the entire flow. @@ -306,19 +356,30 @@ Summary of UX findings from previous steps. An overall accessibility review summary based on the previous steps' reviews. - Summary of accessibility findings from previous steps. + - Summary of accessibility findings from previous steps. -5/10 - This is an overall score for the entire flow. +6/10 - This is an overall score for the entire flow. + + +An overall internationalization review summary based on the previous steps' reviews. + + + - Summary of internationalization findings from previous steps. + + +5/10 - This is an overall score for the entire flow. + + #### Additional Example: - + User Profile Screen. - - + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. @@ -329,8 +390,8 @@ Select the 'Profile' icon (ID: icon_profile) The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all users. - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. + - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. + - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. 5/10 @@ -341,13 +402,26 @@ The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all The 'Profile' icon (ID: icon_profile) has accessibility issues that could affect users with disabilities. - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. + - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. + - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. 4/10 + + +The 'Profile' icon may not be properly adapted for different locales. + + + - Iconography may not be universally recognized - Consider using culturally neutral icons. + - No localization for the tooltip text - Ensure tooltips and labels are localized. + - Date and time formats on the profile screen may not match user locale - Use locale-aware date and time formats. + + +6/10 + + #### Example of Throwing an Informative Error: @@ -359,7 +433,7 @@ Please provide your response below:" `; exports[`PilotPromptCreator should include previous intents in the context 1`] = ` -"# Next Step Generation and UX/Accessibility Reporting +"# Next Step Generation and UX/Accessibility/Internationalization Reporting You are an AI assistant tasked with: @@ -367,7 +441,7 @@ You are an AI assistant tasked with: Please generate a one-line string that precisely describes the next action the user should take to move closer to their goal, and another string (which can be greater than one line) which describes your thoughts while creating the step. If you think that the goal has been reached return a one word action 'success' -2. Creating comprehensive UX and accessibility reports that include a review, findings, and a score. +2. Creating comprehensive UX, Accessibility, and Internationalization reports that include a review, findings, and a score. Please adhere to the instructions below to provide detailed and helpful responses. @@ -412,10 +486,10 @@ Your tasks are as follows: 2. Consider the elements present in the view hierarchy to determine possible next actions. 3. Determine the optimal next action the user should take to move closer to their goal. 4. Ensure the action is directly related to available elements in the view hierarchy. -5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view +5. Make sure to create a unique screen name enclosed with blocks according to the snapshot and view. 6. Generate a one-line string that precisely describes this next action, enclosed within \`\` tags. 7. Provide a detailed description of your thought process enclosed within \`\` tags. -8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX and Accessibility reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. +8. If the goal is achieved, add a \`\` block inside the \`\` section, summarizing the overall flow and findings. Also, provide comprehensive overall UX, Accessibility, and Internationalization reviews with total scores, given all the screens seen in previous steps, inside the respective review sections. 9. For each applicable review section (\`UX\`, \`Accessibility\`, \`Internationalization\`), create a comprehensive report enclosed within corresponding tags (e.g., \`\`, \`\`, \`\`), including a summary, findings, and a score out of 10. 10. Ensure each section is clearly labeled and formatted as shown in the examples. 11. If you cannot determine the next action due to ambiguity or missing information, throw an informative error explaining the problem in one sentence. @@ -429,10 +503,10 @@ If you encounter any issues or have questions, please throw an informative error #### Next Action with Thoughts: - + Registration Page. - - + + To complete the registration process, tapping on the 'Submit' button is necessary. @@ -444,9 +518,9 @@ Tap on the 'Submit' button, which has the ID 'btn_submit' and is located at the The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potentially hindering the registration process. - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. - It uses a color that blends with the background - Use a contrasting color to make the button stand out. - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. + - The button is positioned below multiple input fields, requiring excessive scrolling - Position the 'Submit' button prominently or make it sticky on the screen. + - It uses a color that blends with the background - Use a contrasting color to make the button stand out. + - The button label uses a small font size, which may be hard to read - Increase the font size of the label to improve readability. 6/10 @@ -457,21 +531,34 @@ The 'Submit' button (ID: btn_submit) may not be clearly visible to users, potent The 'Submit' button (ID: btn_submit) lacks essential accessibility features. - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. + - Missing 'aria-label' or accessible name for screen readers. - Add an 'aria-label' with an appropriate description. + - The touch target size is smaller than recommended. - Increase the touch target size to at least 44x44 pixels. + - The contrast ratio between the text and background is insufficient. - Adjust colors to meet contrast ratio guidelines. 5/10 + + +The 'Submit' button may not be properly localized for all supported languages. + + + - The button text is hard-coded in English - Use localization files to support multiple languages. + - Layout may break for languages with longer text strings - Ensure dynamic resizing or text wrapping is implemented. + - Missing support for right-to-left languages - Adjust layout to support RTL languages where necessary. + + +4/10 + + #### Example of Success: - + Goal Page Name. - - + + All actions required to achieve the goal have been completed successfully. An overall summary of the actions taken and reviews conducted during the previous steps. @@ -485,10 +572,10 @@ success An overall UX review summary based on the previous steps' reviews. -Summary of UX findings from previous steps. + - Summary of UX findings from previous steps. -6/10 - This is an overall score for the entire flow. +7/10 - This is an overall score for the entire flow. @@ -496,19 +583,30 @@ Summary of UX findings from previous steps. An overall accessibility review summary based on the previous steps' reviews. - Summary of accessibility findings from previous steps. + - Summary of accessibility findings from previous steps. -5/10 - This is an overall score for the entire flow. +6/10 - This is an overall score for the entire flow. + + +An overall internationalization review summary based on the previous steps' reviews. + + + - Summary of internationalization findings from previous steps. + + +5/10 - This is an overall score for the entire flow. + + #### Additional Example: - + User Profile Screen. - - + + To view the user profile, selecting the 'Profile' icon (ID: icon_profile) is required. @@ -519,8 +617,8 @@ Select the 'Profile' icon (ID: icon_profile) The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all users. - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. + - Uses an uncommon symbol instead of the standard user silhouette - Replace with the standard user silhouette icon. + - Lacks a text label, which may confuse some users - Add a text label or tooltip that says 'Profile'. 5/10 @@ -531,13 +629,26 @@ The 'Profile' icon (ID: icon_profile) might not be immediately recognized by all The 'Profile' icon (ID: icon_profile) has accessibility issues that could affect users with disabilities. - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. + - No 'aria-label' provided for screen readers - Add an 'aria-label' with the text 'User Profile'. + - The icon is not reachable via keyboard navigation - Ensure the icon can be focused and activated via keyboard. 4/10 + + +The 'Profile' icon may not be properly adapted for different locales. + + + - Iconography may not be universally recognized - Consider using culturally neutral icons. + - No localization for the tooltip text - Ensure tooltips and labels are localized. + - Date and time formats on the profile screen may not match user locale - Use locale-aware date and time formats. + + +6/10 + + #### Example of Throwing an Informative Error: diff --git a/src/utils/extractOutputs.ts b/src/utils/extractOutputs.ts index 8624d90..57a8c76 100644 --- a/src/utils/extractOutputs.ts +++ b/src/utils/extractOutputs.ts @@ -12,11 +12,12 @@ export const OUTPUTS_MAPPINGS: Record = { score: { tag: "SCORE", isRequired: false }, }, PILOT_STEP: { - screenName: { tag: "SCREENNAME", isRequired: true }, + screenDescription: { tag: "SCREENDESCRIPTION", isRequired: true }, thoughts: { tag: "THOUGHTS", isRequired: true }, action: { tag: "ACTION", isRequired: true }, ux: { tag: "UX", isRequired: false }, a11y: { tag: "ACCESSIBILITY", isRequired: false }, + i18n: { tag: "INTERNATIONALIZATION", isRequired: false }, }, PILOT_SUMMARY: { summary: { tag: "SUMMARY", isRequired: true }, diff --git a/src/utils/logger/index.ts b/src/utils/logger/index.ts index a946045..35c64bf 100644 --- a/src/utils/logger/index.ts +++ b/src/utils/logger/index.ts @@ -69,8 +69,8 @@ class Logger { .join(" "); } - private formatMessage(...components: LoggerMessageComponent[]): string { - return components + private formatMessageForLogFile(level: string, ...components: LoggerMessageComponent[]): string { + const messageText = components .map((component) => { if (typeof component === "string") { return component; @@ -85,9 +85,12 @@ class Logger { return message; }) .join(" "); + const timestamp: string = this.formatMessageEntryForLogFile(new Date()); + const levelUpper: string = level.toUpperCase(); + return `[${timestamp}] ${levelUpper}: ${messageText}`; } - private formatTimestamp(date: Date): string { + private formatMessageEntryForLogFile(date: Date): string { const pad = (n: number) => (n < 10 ? "0" + n : n.toString()); const year = date.getFullYear(); const month = pad(date.getMonth() + 1); @@ -114,15 +117,8 @@ class Logger { return component; }); - this.logger[level](this.colorizeMessage(...newComponents)); - - const timestamp = this.formatTimestamp(new Date()); - const levelUpper = level.toUpperCase(); - const messageText = this.formatMessage(...components); - const logEntry = `[${timestamp}] ${levelUpper}: ${messageText}`; - - this.logs.push(logEntry); + this.logs.push(this.formatMessageForLogFile(level, ...components)); } public info(...components: LoggerMessageComponent[]): void { @@ -176,7 +172,7 @@ class Logger { public writeLogsToFile(filename: string): void { try { fs.writeFileSync(filename, this.logs.join("\n"), "utf8"); - this.info(`Logs have been written to ${filename}`); + this.info(`💾 Logs have been written to ${filename}`); } catch (err) { this.error("Failed to write logs to file:", { message: `${(err as Error).message}`, From 6a8c13d170cc87f1ebeaf2c97f6ff3f72e5eb1f6 Mon Sep 17 00:00:00 2001 From: Tzviel Solodar Date: Mon, 27 Jan 2025 13:34:00 +0200 Subject: [PATCH 4/4] apply lint to all files --- src/Copilot.test.ts | 44 +++++++++---------- src/actions/PilotPerformer.test.ts | 2 +- src/actions/PilotPerformer.ts | 24 ++++++---- src/drivers/playwright/index.ts | 66 +++++++++++++++++----------- src/integration tests/index.test.ts | 1 - src/types/framework.ts | 2 +- src/types/pilot.ts | 2 +- src/utils/PilotPromptCreator.test.ts | 1 - src/utils/PilotPromptCreator.ts | 2 +- src/utils/logger/index.ts | 28 ++++++------ 10 files changed, 94 insertions(+), 78 deletions(-) diff --git a/src/Copilot.test.ts b/src/Copilot.test.ts index 321fa9b..bc23f0f 100644 --- a/src/Copilot.test.ts +++ b/src/Copilot.test.ts @@ -6,7 +6,6 @@ import { ScreenCapturerResult, PromptHandler, PilotStepReport, - PilotReport, } from "@/types"; import { mockCache, mockedCacheFile } from "./test-utils/cache"; import { ScreenCapturer } from "@/utils/ScreenCapturer"; @@ -299,11 +298,12 @@ describe("Copilot", () => { instance.extendAPICatalog([barCategory1]); instance.extendAPICatalog([barCategory2], dummyContext); - expect(mockConfig.frameworkDriver.apiCatalog.categories.length).toEqual(1); - expect(mockConfig.frameworkDriver.apiCatalog.categories[0].items).toEqual([ - ...barCategory1.items, - ...barCategory2.items, - ]); + expect(mockConfig.frameworkDriver.apiCatalog.categories.length).toEqual( + 1, + ); + expect(mockConfig.frameworkDriver.apiCatalog.categories[0].items).toEqual( + [...barCategory1.items, ...barCategory2.items], + ); expect(spyCopilotStepPerformer).toHaveBeenCalledWith(dummyContext); }); @@ -407,22 +407,20 @@ describe("Copilot", () => { summary: "All was good", }; - jest - .spyOn(instance["pilotPerformer"], "perform") - .mockResolvedValue({ - summary: pilotOutputSuccess.summary, - goal: goal, - steps: [ - { - screenDescription: pilotOutputStep1.screenDescription, - plan: pilotOutputStep1.plan, - code: "code executed", - review: pilotOutputStep1.review, - goalAchieved: pilotOutputStep1.goalAchieved, - }, - ], - review: pilotOutputSuccess.review, - }); + jest.spyOn(instance["pilotPerformer"], "perform").mockResolvedValue({ + summary: pilotOutputSuccess.summary, + goal: goal, + steps: [ + { + screenDescription: pilotOutputStep1.screenDescription, + plan: pilotOutputStep1.plan, + code: "code executed", + review: pilotOutputStep1.review, + goalAchieved: pilotOutputStep1.goalAchieved, + }, + ], + review: pilotOutputSuccess.review, + }); const result = await instance.pilot(goal); @@ -443,4 +441,4 @@ describe("Copilot", () => { }); }); }); -}); \ No newline at end of file +}); diff --git a/src/actions/PilotPerformer.test.ts b/src/actions/PilotPerformer.test.ts index 522ac17..9c69d24 100644 --- a/src/actions/PilotPerformer.test.ts +++ b/src/actions/PilotPerformer.test.ts @@ -445,4 +445,4 @@ describe("PilotPerformer", () => { expect(result).toEqual(expectedReport); }); }); -}); \ No newline at end of file +}); diff --git a/src/actions/PilotPerformer.ts b/src/actions/PilotPerformer.ts index f372412..08a386c 100644 --- a/src/actions/PilotPerformer.ts +++ b/src/actions/PilotPerformer.ts @@ -14,7 +14,6 @@ import { import { extractOutputs, OUTPUTS_MAPPINGS } from "@/utils/extractOutputs"; import { CopilotStepPerformer } from "@/actions/CopilotStepPerformer"; import { ScreenCapturer } from "@/utils/ScreenCapturer"; -import fs from 'fs'; import logger from "@/utils/logger"; export class PilotPerformer { @@ -40,7 +39,10 @@ export class PilotPerformer { }; } - private logReviewSection(review: PilotReviewSection, type: "ux" | "a11y" | "i18n") { + private logReviewSection( + review: PilotReviewSection, + type: "ux" | "a11y" | "i18n", + ) { const config: { [key: string]: { emoji: string; @@ -102,10 +104,11 @@ export class PilotPerformer { const generatedPilotTaskDetails: string = await this.promptHandler.runPrompt(prompt, snapshot); - const { screenDescription, thoughts, action, ux, a11y, i18n } = extractOutputs({ - text: generatedPilotTaskDetails, - outputsMapper: OUTPUTS_MAPPINGS.PILOT_STEP, - }); + const { screenDescription, thoughts, action, ux, a11y, i18n } = + extractOutputs({ + text: generatedPilotTaskDetails, + outputsMapper: OUTPUTS_MAPPINGS.PILOT_STEP, + }); analysisLoggerSpinner.stop("success", `💭 Thoughts:`, { message: thoughts, @@ -123,7 +126,7 @@ export class PilotPerformer { logger.info({ message: `Conducting review for ${screenDescription}\n`, isBold: true, - color: 'whiteBright', + color: "whiteBright", }); review.ux && this.logReviewSection(review.ux, "ux"); @@ -194,7 +197,10 @@ export class PilotPerformer { screenCapture, ); copilotSteps = [...copilotSteps, { step: plan.action, code, result }]; - previousSteps = [...previousSteps, { screenDescription, step: plan.action, review }]; + previousSteps = [ + ...previousSteps, + { screenDescription, step: plan.action, review }, + ]; const stepReport: PilotStepReport = { screenDescription, @@ -213,4 +219,4 @@ export class PilotPerformer { logger.writeLogsToFile(`pilot_logs_${Date.now()}`); return report; } -} \ No newline at end of file +} diff --git a/src/drivers/playwright/index.ts b/src/drivers/playwright/index.ts index fc7e5c1..4d675ae 100644 --- a/src/drivers/playwright/index.ts +++ b/src/drivers/playwright/index.ts @@ -82,7 +82,9 @@ export class PlaywrightFrameworkDriver implements TestingFrameworkDriver { playwright, expect: playwrightExpect, }, - restrictions:['Never use expect on the page it self for example : await expect(page).toBeVisible()'], + restrictions: [ + "Never use expect on the page it self for example : await expect(page).toBeVisible()", + ], categories: [ { title: "Page Management", @@ -91,7 +93,7 @@ export class PlaywrightFrameworkDriver implements TestingFrameworkDriver { signature: "getCurrentPage(): playwright.Page | undefined", description: "Gets the current active page instance.", example: "const page = getCurrentPage();", - guidelines: [ + guidelines: [ "Always check if page exists before operations.", "Returns undefined if no page is set.", "Use before any page interactions.", @@ -102,7 +104,7 @@ export class PlaywrightFrameworkDriver implements TestingFrameworkDriver { description: "Sets the current active page for interactions.", example: "const page = await context.newPage(); setCurrentPage(page);", - guidelines: [ + guidelines: [ "Must be called after creating a new page.", "Required before any page interactions.", "Only one page can be active at a time.", @@ -127,7 +129,7 @@ setCurrentPage(page); await page.goto('https://www.test.com/'); await page.waitForLoadState('load') `, - guidelines: [ + guidelines: [ "Set longer timeouts (30s or more) to handle slow operations.", "Can use chromium, firefox, or webkit browsers.", "Remember to call setCurrentPage after creating a page.", @@ -182,7 +184,7 @@ if (page) { if (page) { await page.reload(); }`, - guidelines: [ + guidelines: [ "Avoid explicit waits - let assertions handle timing.", "Good for refreshing stale content.", ], @@ -199,7 +201,7 @@ if (page) { if (page) { await page.getByRole('button', { name: 'Submit' }).click(); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Preferred way to locate interactive elements.", "Improves test accessibility coverage.", @@ -212,7 +214,7 @@ if (page) { if (page) { await page.getByText('Welcome').isVisible(); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Good for finding visible text on page.", "Can use exact or fuzzy matching.", @@ -225,7 +227,7 @@ if (page) { if (page) { await page.getByLabel('Username').fill('john'); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Best practice for form inputs.", "More reliable than selectors.", @@ -243,7 +245,7 @@ if (page) { if (page) { await page.getByRole('button').click(); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Automatically waits for element.", "Handles scrolling automatically.", @@ -256,7 +258,7 @@ if (page) { if (page) { await page.getByLabel('Password').fill('secret'); }`, - guidelines: [ + guidelines: [ "Always check if page exists first.", "Preferred over type() for forms.", "Clears existing value first.", @@ -320,7 +322,8 @@ if (page) { }, { signature: "await locator.scrollIntoViewIfNeeded()", - description: "Scrolls the element into view if it is not already visible.", + description: + "Scrolls the element into view if it is not already visible.", example: `const page = getCurrentPage(); if (page) { await page.getByText('Load More').scrollIntoViewIfNeeded(); @@ -523,7 +526,8 @@ if (page) { ], }, { - signature: "const elementHandles = await locator.elementHandles()", + signature: + "const elementHandles = await locator.elementHandles()", description: "Gets all matching element handles.", example: `const page = getCurrentPage(); if (page) { @@ -570,7 +574,8 @@ if (page) { }, { signature: "await expect(locator).toHaveText(expectedText)", - description: "Asserts that the element's text content matches the expected text.", + description: + "Asserts that the element's text content matches the expected text.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByTestId('username')).toHaveText('John Doe'); @@ -583,7 +588,8 @@ if (page) { }, { signature: "await expect(page).toHaveURL(expectedURL)", - description: "Asserts that the page's URL matches the expected URL.", + description: + "Asserts that the page's URL matches the expected URL.", example: `const page = getCurrentPage(); if (page) { await expect(page).toHaveURL('https://www.example.com/dashboard'); @@ -596,7 +602,8 @@ if (page) { }, { signature: "await expect(locator).toHaveValue(expectedValue)", - description: "Asserts that an input element has the expected value.", + description: + "Asserts that an input element has the expected value.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByPlaceholder('Email')).toHaveValue('user@example.com'); @@ -635,7 +642,8 @@ if (page) { }, { signature: "await expect(locator).toBeChecked()", - description: "Asserts that a checkbox or radio button is checked.", + description: + "Asserts that a checkbox or radio button is checked.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByLabel('Accept Terms')).toBeChecked(); @@ -648,7 +656,8 @@ if (page) { }, { signature: "await expect(locator).toBeEditable()", - description: "Asserts that an input or textarea element is editable.", + description: + "Asserts that an input or textarea element is editable.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByPlaceholder('Enter your name')).toBeEditable(); @@ -661,7 +670,8 @@ if (page) { }, { signature: "await expect(locator).toHaveAttribute(name, value)", - description: "Asserts that the element has the specified attribute value.", + description: + "Asserts that the element has the specified attribute value.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByTestId('profile-link')).toHaveAttribute('href', '/profile'); @@ -674,7 +684,8 @@ if (page) { }, { signature: "await expect(locator).toHaveClass(expectedClass)", - description: "Asserts that the element has the specified CSS class.", + description: + "Asserts that the element has the specified CSS class.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByRole('button', { name: 'Submit' })).toHaveClass('btn-primary'); @@ -687,7 +698,8 @@ if (page) { }, { signature: "await expect(locator).toContainText(expectedText)", - description: "Asserts that the element's text contains the expected text.", + description: + "Asserts that the element's text contains the expected text.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByTestId('notification')).toContainText('Success'); @@ -700,7 +712,8 @@ if (page) { }, { signature: "await expect(locator).toHaveJSProperty(name, value)", - description: "Asserts that the element has the specified JavaScript property.", + description: + "Asserts that the element has the specified JavaScript property.", example: `const page = getCurrentPage(); if (page) { await expect(page.getByTestId('toggle')).toHaveJSProperty('checked', true); @@ -726,7 +739,8 @@ if (page) { }, { signature: "await expect(page).toHaveTitle(expectedTitle)", - description: "Asserts that the page's title matches the expected title.", + description: + "Asserts that the page's title matches the expected title.", example: `const page = getCurrentPage(); if (page) { await expect(page).toHaveTitle('Dashboard - MyApp'); @@ -739,7 +753,8 @@ if (page) { }, { signature: "await expect(page).toHaveScreenshot([options])", - description: "Asserts that the page's screenshot matches a stored reference image.", + description: + "Asserts that the page's screenshot matches a stored reference image.", example: `const page = getCurrentPage(); if (page) { await expect(page).toHaveScreenshot('dashboard.png'); @@ -752,7 +767,8 @@ if (page) { }, { signature: "await expect(locator).toHaveCount(expectedCount)", - description: "Asserts that the locator resolves to a specific number of elements.", + description: + "Asserts that the locator resolves to a specific number of elements.", example: `const page = getCurrentPage(); if (page) { await expect(page.locator('.todo-item')).toHaveCount(3); @@ -777,7 +793,7 @@ if (page) { ], }, ], - } + }, ], }; } diff --git a/src/integration tests/index.test.ts b/src/integration tests/index.test.ts index aa4a460..93ba8c3 100644 --- a/src/integration tests/index.test.ts +++ b/src/integration tests/index.test.ts @@ -562,7 +562,6 @@ describe("Copilot Integration Tests", () => { }); }); - describe("Cache Modes", () => { beforeEach(() => { mockPromptHandler.runPrompt.mockResolvedValue("// No operation"); diff --git a/src/types/framework.ts b/src/types/framework.ts index 36250e5..41a0ebb 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -32,7 +32,7 @@ export type TestingFrameworkAPICatalog = { context: any; /** Available API method categories */ categories: TestingFrameworkAPICatalogCategory[]; - /** List of restrictions and guidlines of wrong actions */ + /** List of restrictions and guidlines of wrong actions */ restrictions?: string[]; }; diff --git a/src/types/pilot.ts b/src/types/pilot.ts index fa34ed7..58e5b46 100644 --- a/src/types/pilot.ts +++ b/src/types/pilot.ts @@ -11,7 +11,7 @@ export type PreviousStep = { }; /** Review section types */ -export type PilotReviewSectionType = "ux" | "a11y"| "i18n"; +export type PilotReviewSectionType = "ux" | "a11y" | "i18n"; /** Complete pilot review */ export type PilotReview = { diff --git a/src/utils/PilotPromptCreator.test.ts b/src/utils/PilotPromptCreator.test.ts index 5b94166..7878f56 100644 --- a/src/utils/PilotPromptCreator.test.ts +++ b/src/utils/PilotPromptCreator.test.ts @@ -22,7 +22,6 @@ describe("PilotPromptCreator", () => { { screenDescription: "default 1", step: "navigate to login screen", - }, { screenDescription: "default 2", diff --git a/src/utils/PilotPromptCreator.ts b/src/utils/PilotPromptCreator.ts index ee4bb0d..754ff75 100644 --- a/src/utils/PilotPromptCreator.ts +++ b/src/utils/PilotPromptCreator.ts @@ -342,4 +342,4 @@ The 'Profile' icon may not be properly adapted for different locales. ]; return steps; } -} \ No newline at end of file +} diff --git a/src/utils/logger/index.ts b/src/utils/logger/index.ts index 35c64bf..f27fc2a 100644 --- a/src/utils/logger/index.ts +++ b/src/utils/logger/index.ts @@ -12,7 +12,7 @@ import { LoggerMessageColor, LoggerOperationResultType, } from "@/types/logger"; -import * as fs from "fs"; +import * as fs from "fs"; class Logger { private static instance: Logger; @@ -27,7 +27,7 @@ class Logger { error: "red", debug: "gray", }; - private logs: string[] = []; + private logs: string[] = []; private constructor() { this.logger = createLogger({ @@ -69,7 +69,10 @@ class Logger { .join(" "); } - private formatMessageForLogFile(level: string, ...components: LoggerMessageComponent[]): string { + private formatMessageForLogFile( + level: string, + ...components: LoggerMessageComponent[] + ): string { const messageText = components .map((component) => { if (typeof component === "string") { @@ -85,15 +88,15 @@ class Logger { return message; }) .join(" "); - const timestamp: string = this.formatMessageEntryForLogFile(new Date()); - const levelUpper: string = level.toUpperCase(); - return `[${timestamp}] ${levelUpper}: ${messageText}`; + const timestamp: string = this.formatMessageEntryForLogFile(new Date()); + const levelUpper: string = level.toUpperCase(); + return `[${timestamp}] ${levelUpper}: ${messageText}`; } private formatMessageEntryForLogFile(date: Date): string { const pad = (n: number) => (n < 10 ? "0" + n : n.toString()); const year = date.getFullYear(); - const month = pad(date.getMonth() + 1); + const month = pad(date.getMonth() + 1); const day = pad(date.getDate()); const hours = pad(date.getHours()); const minutes = pad(date.getMinutes()); @@ -137,9 +140,7 @@ class Logger { this.log("debug", ...components); } - public startSpinner( - ...components: LoggerMessageComponent[] - ): LoggerSpinner { + public startSpinner(...components: LoggerMessageComponent[]): LoggerSpinner { const spinner = ora(this.colorizeMessage(...components)).start(); const stop = ( @@ -149,10 +150,7 @@ class Logger { spinner.prefixText = ""; const message = this.colorizeMessage(...components); - const spinnerActions: Record< - LoggerOperationResultType, - () => ora.Ora - > = { + const spinnerActions: Record ora.Ora> = { success: () => spinner.succeed(message), failure: () => spinner.fail(message), warn: () => spinner.warn(message), @@ -183,4 +181,4 @@ class Logger { } } -export default Logger.getInstance(); \ No newline at end of file +export default Logger.getInstance();