Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: stabilize playwright driver + add screen name to pilot's steps … #67

Merged
merged 5 commits into from
Jan 27, 2025
Merged

feat: stabilize playwright driver + add screen name to pilot's steps … #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0c02b2b
feat: stabilize playwright driver + add screen name to pilot's steps …
tzvielwix Jan 26, 2025
fcfd947
wip: add screen name to prompt creator
tzvielwix Jan 26, 2025
20d6945
fixes after review + add i18n to pilot review options
tzvielwix Jan 27, 2025
08251e5
Merge branch 'master' into stabilzie-playwright
tzvielwix Jan 27, 2025
6a8c13d
apply lint to all files
tzvielwix Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 14 additions & 6 deletions src/Copilot.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,9 +326,11 @@ describe("Copilot", () => {
const goal = "test goal";

const mockPilotResult = {
summary: "Test completed successfully",
goal,
steps: [
{
screenDescription: "Screen 1",
plan: {
thoughts: "Step 1 thoughts",
action: "Tap on GREAT button",
Expand All @@ -337,6 +339,7 @@ describe("Copilot", () => {
goalAchieved: false,
},
{
screenDescription: "Screen 2",
plan: {
thoughts: "Completed successfully",
action: "success",
Expand All @@ -362,9 +365,10 @@ describe("Copilot", () => {
const goal = "Test the login flow";

const pilotOutputStep1: PilotStepReport = {
screenDescription: "Login Screen",
plan: {
thoughts: "Step 1 thoughts",
action: "Tap on GREAT button",
action: "Tap on Login button",
},
review: {
ux: {
Expand All @@ -382,8 +386,9 @@ describe("Copilot", () => {
};

const pilotOutputSuccess: PilotStepReport = {
screenDescription: "Home Screen",
plan: {
thoughts: "Completed successfully <SUMMARY> all was good </SUMMARY>",
thoughts: "Completed successfully",
action: "success",
},
review: {
Expand All @@ -399,17 +404,19 @@ describe("Copilot", () => {
},
},
goalAchieved: true,
summary: "All was good",
};

jest.spyOn(instance["pilotPerformer"], "perform").mockResolvedValue({
summary: "all was good",
summary: pilotOutputSuccess.summary,
goal: goal,
steps: [
{
screenDescription: pilotOutputStep1.screenDescription,
plan: pilotOutputStep1.plan,
code: "code executed",
review: pilotOutputStep1.review,
goalAchieved: true,
goalAchieved: pilotOutputStep1.goalAchieved,
},
],
review: pilotOutputSuccess.review,
Expand All @@ -419,14 +426,15 @@ describe("Copilot", () => {

expect(instance["pilotPerformer"].perform).toHaveBeenCalledWith(goal);
expect(result).toEqual({
summary: "all was good",
summary: pilotOutputSuccess.summary,
goal: goal,
steps: [
{
screenDescription: pilotOutputStep1.screenDescription,
plan: pilotOutputStep1.plan,
code: "code executed",
review: pilotOutputStep1.review,
goalAchieved: true,
goalAchieved: pilotOutputStep1.goalAchieved,
},
],
review: pilotOutputSuccess.review,
Expand Down
84 changes: 76 additions & 8 deletions src/actions/PilotPerformer.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { PilotPerformer } from "@/actions/PilotPerformer";
import { PilotPromptCreator } from "@/utils/PilotPromptCreator";
import { ScreenCapturer } from "@/utils/ScreenCapturer";
import {
PreviousStep,
PilotPreviousStep,
PromptHandler,
ScreenCapturerResult,
PilotStepReport,
Expand All @@ -14,8 +14,11 @@ const GOAL = "tap button";
const VIEW_HIERARCHY = "<view></view>";
const GENERATED_PROMPT = "generated prompt";

// Updated PROMPT_RESULT to include UX and Accessibility sections
// Updated PROMPT_RESULT to include screenDescription, UX, Accessibility, and Internationalization sections
const PROMPT_RESULT = `
<SCREENDESCRIPTION>
default name
</SCREENDESCRIPTION>
<THOUGHTS>
I think this is great
</THOUGHTS>
Expand Down Expand Up @@ -45,7 +48,19 @@ The review of accessibility
<SCORE>
8/10
</SCORE>
</ACCESSIBILITY>`;
</ACCESSIBILITY>
<INTERNATIONALIZATION>
<SUMMARY>
The review of i18n
</SUMMARY>
<FINDINGS>
- i18n finding one
- i18n finding two
</FINDINGS>
<SCORE>
6/10
</SCORE>
</INTERNATIONALIZATION>`;

const SNAPSHOT_DATA = "snapshot_data";

Expand Down Expand Up @@ -79,12 +94,12 @@ describe("PilotPerformer", () => {
capture: jest.fn(),
} as unknown as jest.Mocked<ScreenCapturer>;

// Instantiate PilotPerformer with the mocks, including the capture function
// Instantiate PilotPerformer with the mocks
pilotPerformer = new PilotPerformer(
mockPromptCreator,
mockCopilotStepPerformer,
mockPromptHandler,
mockScreenCapturer, // Pass the mock capture function
mockScreenCapturer,
);
});

Expand Down Expand Up @@ -125,6 +140,7 @@ describe("PilotPerformer", () => {
);

const expectedResult = {
screenDescription: "default name",
plan: {
thoughts: "I think this is great",
action: "Tap on GREAT button",
Expand All @@ -140,6 +156,11 @@ describe("PilotPerformer", () => {
findings: ["ACC finding one", "ACC finding two"],
score: "8/10",
},
i18n: {
summary: "The review of i18n",
findings: ["i18n finding one", "i18n finding two"],
score: "6/10",
},
},
goalAchieved: false,
};
Expand Down Expand Up @@ -167,6 +188,7 @@ describe("PilotPerformer", () => {
);

const expectedResult = {
screenDescription: "default name",
plan: {
thoughts: "I think this is great",
action: "Tap on GREAT button",
Expand All @@ -182,6 +204,11 @@ describe("PilotPerformer", () => {
findings: ["ACC finding one", "ACC finding two"],
score: "8/10",
},
i18n: {
summary: "The review of i18n",
findings: ["i18n finding one", "i18n finding two"],
score: "6/10",
},
},
goalAchieved: false,
};
Expand Down Expand Up @@ -209,6 +236,7 @@ describe("PilotPerformer", () => {
);

const expectedResult = {
screenDescription: "default name",
plan: {
thoughts: "I think this is great",
action: "Tap on GREAT button",
Expand All @@ -224,6 +252,11 @@ describe("PilotPerformer", () => {
findings: ["ACC finding one", "ACC finding two"],
score: "8/10",
},
i18n: {
summary: "The review of i18n",
findings: ["i18n finding one", "i18n finding two"],
score: "6/10",
},
},
goalAchieved: false,
};
Expand All @@ -243,11 +276,27 @@ describe("PilotPerformer", () => {

it("should perform an intent successfully with previous intents", async () => {
const intent = "current intent";
const previousIntents: PreviousStep[] = [
const previousIntents: PilotPreviousStep[] = [
{
screenDescription: "default",
step: "previous intent",
code: "previous code",
result: "previous result",
review: {
ux: {
summary: "Previous UX summary",
findings: ["Previous UX finding"],
score: "6/10",
},
a11y: {
summary: "Previous Accessibility summary",
findings: ["Previous ACC finding"],
score: "7/10",
},
i18n: {
summary: "Previous i18n summary",
findings: ["Previous i18n finding"],
score: "5/10",
},
},
},
];

Expand All @@ -260,6 +309,7 @@ describe("PilotPerformer", () => {
);

const expectedResult = {
screenDescription: "default name",
plan: {
thoughts: "I think this is great",
action: "Tap on GREAT button",
Expand All @@ -275,6 +325,11 @@ describe("PilotPerformer", () => {
findings: ["ACC finding one", "ACC finding two"],
score: "8/10",
},
i18n: {
summary: "The review of i18n",
findings: ["i18n finding one", "i18n finding two"],
score: "6/10",
},
},
goalAchieved: false,
};
Expand All @@ -295,6 +350,7 @@ describe("PilotPerformer", () => {
describe("perform", () => {
it("should perform multiple steps until success is returned", async () => {
const pilotOutputStep1: PilotStepReport = {
screenDescription: "Screen 1",
plan: {
thoughts: "Step 1 thoughts",
action: "Tap on GREAT button",
Expand All @@ -310,11 +366,17 @@ describe("PilotPerformer", () => {
findings: [],
score: "8/10",
},
i18n: {
summary: "i18n review for step 1",
findings: [],
score: "6/10",
},
},
goalAchieved: false,
};

const pilotOutputSuccess: PilotStepReport = {
screenDescription: "Screen 2",
plan: {
thoughts: "Completed successfully",
action: "success",
Expand All @@ -330,6 +392,11 @@ describe("PilotPerformer", () => {
findings: [],
score: "9/10",
},
i18n: {
summary: "Final i18n review",
findings: [],
score: "8/10",
},
},
goalAchieved: true,
summary: "all was good",
Expand Down Expand Up @@ -365,6 +432,7 @@ describe("PilotPerformer", () => {
goal: GOAL,
steps: [
{
screenDescription: "Screen 1",
plan: pilotOutputStep1.plan,
code: "code executed",
review: pilotOutputStep1.review,
Expand Down
Loading
Loading