From b6974a1f77ada12de588ec998ce302b28fe7beb5 Mon Sep 17 00:00:00 2001 From: Shyam Raghuwanshi Date: Wed, 14 Aug 2024 22:41:12 +0530 Subject: [PATCH] Updating rpa challenge and playwright (#406) * updating rpa challenge and playwright * updating-secrets --- JS/edgechains/arakoodev/package.json | 3 +- .../arakoodev/src/ai/src/lib/openai/openai.ts | 2 +- .../src/scraper/src/actions/index.ts | 325 ++++++++++++++++ .../src/scraper/src/lib/playwright.ts | 359 +++++++----------- .../arakoodev/src/scraper/src/utils/index.ts | 11 +- .../rpa-challenge/jsonnet/main.jsonnet | 69 +++- .../rpa-challenge/jsonnet/secrets.jsonnet | 1 - .../examples/rpa-challenge/package.json | 2 + .../examples/rpa-challenge/src/index.ts | 10 +- .../src/lib/downloadExcelTask.cts | 18 + .../rpa-challenge/src/lib/generateTask.cts | 25 ++ .../rpa-challenge/src/lib/playwright.cts | 12 + .../rpa-challenge/src/lib/playwright.ts | 12 - package.json | 2 + 14 files changed, 615 insertions(+), 236 deletions(-) create mode 100644 JS/edgechains/arakoodev/src/scraper/src/actions/index.ts create mode 100644 JS/edgechains/examples/rpa-challenge/src/lib/downloadExcelTask.cts create mode 100644 JS/edgechains/examples/rpa-challenge/src/lib/generateTask.cts create mode 100644 JS/edgechains/examples/rpa-challenge/src/lib/playwright.cts delete mode 100644 JS/edgechains/examples/rpa-challenge/src/lib/playwright.ts diff --git a/JS/edgechains/arakoodev/package.json b/JS/edgechains/arakoodev/package.json index 39739a2f..602c1800 100644 --- a/JS/edgechains/arakoodev/package.json +++ b/JS/edgechains/arakoodev/package.json @@ -28,7 +28,7 @@ "@lifeomic/attempt": "^3.1.0", "@playwright/test": "^1.45.3", "@supabase/supabase-js": "^2.42.3", - "axios": "^1.7.2", + "axios": "^1.7.4", "axios-retry": "^4.1.0", "cheerio": "^1.0.0-rc.12", "cors": "^2.8.5", @@ -42,6 +42,7 @@ "jsdom": "^24.1.0", "node-fetch": "^3.3.2", "node-html-parser": "^6.1.13", + "openai": "^4.55.3", "pdf-parse": "^1.1.1", "pg": "^8.11.5", "playwright": "^1.45.1", diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/openai/openai.ts b/JS/edgechains/arakoodev/src/ai/src/lib/openai/openai.ts index 91bc1af6..0eaf85f6 100644 --- a/JS/edgechains/arakoodev/src/ai/src/lib/openai/openai.ts +++ b/JS/edgechains/arakoodev/src/ai/src/lib/openai/openai.ts @@ -118,7 +118,7 @@ export class OpenAI { }, ] : chatOptions.messages, - max_tokens: chatOptions.max_tokens || 256, + max_tokens: chatOptions.max_tokens || 1024, temperature: chatOptions.temperature || 0.7, functions: chatOptions.functions, function_call: chatOptions.function_call || "auto", diff --git a/JS/edgechains/arakoodev/src/scraper/src/actions/index.ts b/JS/edgechains/arakoodev/src/scraper/src/actions/index.ts new file mode 100644 index 00000000..c2dd5c0d --- /dev/null +++ b/JS/edgechains/arakoodev/src/scraper/src/actions/index.ts @@ -0,0 +1,325 @@ +import { expect, Page } from '@playwright/test'; +import { + parseSite, +} from "../utils/index.js"; +import { removeBlankTags } from '../utils/page-parser.js'; + +const AsyncFunction = async function () { }.constructor; + +export async function generatePrompt(page: Page, userTask: string) { + const [currentPageUrl, currentPageTitle, siteOverview] = await Promise.all([ + page.evaluate("location.href"), + page.evaluate("document.title"), + parseSite(page).then((html) => removeBlankTags(html).slice(0, 20000)), + ]).catch((error: any) => { + console.log(error.message + " " + error.stack); + return error.message + " " + error.stack; + }); + + const systemPrompt = ` + You are a Senior SDET tasked with writing Playwright code for testing purposes. Your role involves implementing specific task-based code segments within a larger test file, following the instructions provided closely. Assume that common imports like 'test' and 'expect' from '@playwright/test' are already at the top of the file. + + Context: + - Your computer is a Mac. Cmd is the meta key, META. + - The browser is already open. + - Current page URL: ${currentPageUrl}. + - Current page title: ${currentPageTitle}. + - Overview of the site in HTML format: + \`\`\` + ${siteOverview} + \`\`\` + + Key Points: + - Start directly with Playwright actions as described in the user task, without adding extraneous steps or assertions. + - Include assertions like 'expect' statements or wait functions such as 'waitForLoadState' only when they are specifically requested in the user task. + - Minimal, relevant comments should be used to clarify complex actions or essential aspects of the test's purpose. + - Apply 'frameLocator' for content in nested iframes, as needed based on the task requirements. + + User Task: ${userTask} + + Expected Code Format: + \`\`\` + // [Insert Playwright code based on the task description. Begin with necessary actions directly, and include 'waitForLoadState', assertions, or 'expect' statements only if explicitly requested in the task. Comments should be concise and pertinent, especially for complex actions or decisions.] + \`\`\` + + The objective is to create Playwright code that is efficient, precise, and perfectly aligned with the task's requirements, integrating seamlessly into the larger test file. All actions and comments should be relevant and necessary, catering to a senior-level professional's understanding of the testing scenario.`; + + return systemPrompt; + +} + + +export function generateTaskArrPrompt(task: string) { + return ` + Given the following task description: + + ${task} + + Extract the key actions from this task and return them as an array of strings. Each action should be a separate string in the array. If the task description contains syntax errors or you think a command can be improved for better clarity and effectiveness, please make the necessary corrections and improvements. For example: + + Input: + "Go to Hacker News and click on the first link. Then give me all the text of this page." + + Response Format: + \`\`\` + {[ + "Navigate to the Hacker News website by entering the URL 'https://news.ycombinator.com/' in the browser", + "Identify and click on the first link displayed on the Hacker News homepage", + "Extract all the text from the page", + "Return that containt using return statement" + ]} + \`\`\` + + Input: + "Go to random website. we have fields in this pattern First Name Last Name Company Name Role in Company Address Email Phone Number and then fill this fields John Smith IT Solutions Analyst 98 North Road " + + 'Response Format: + \`\`\` + {[ + "Navigate to the random website by entering the URL 'https://www.randomwebsite.com/' in the browser", + "Fill the form fields with the following data: First Name: John, Last Name: Smith, Company Name: IT Solutions, Role in Company: Analyst, Address: 98 North Road, Email:", + ]} + \`\`\` + + Input: + "Go to google and search for the term 'automation'. Click on the first link and extract the text from the page." + + Response Format: + \`\`\` + {[ + "Navigate to the random website by entering the URL 'https://google.com' in the browser", + "Search for the term 'automation' in the search bar and hit Enter key", + "Click on the first link displayed in the search results", + "Extract all the text from the page", + "Return that containt using return statement" + ]} + \`\`\` + \n + Ensure that each action is specific, clear, and comprehensive to facilitate precise implementation. + `; +} + + +export function handleHistory(goalsArr: string[]): { role: string, content: string }[] { + let goalsStr = '' + for (let index = 0; index < goalsArr.length; index++) { + const element = goalsArr[index]; + goalsStr += `${index + 1}. Goal ${index + 1}: ${element}\n` + } + + return [{ + role: 'system', + content: `You are a Senior SDET tasked with writing Playwright code for testing purposes. Your role + involves implementing specific task-based code segments within a larger test file, following the + instructions provided closely. Assume that common imports like 'test' and 'expect' from '@playwright/test' are already at the top of the file.\n` + + 'Your decisions must always be made independently without seeking user assistance. Play to your strengths as an LLM and pursue simple strategies with no legal complications. We have three function to perform tasks "goToLink", "intractWithPage" and "findInPage". If any task have need to go to any url use or need go to a specific url use "goToLink" command. If any task takes more then 3 requests, use the "findInPage" command so we can try searching for the element using a function but do not use "findInPage" command unnecessary it has a cost.Otherwise use intractWithPage most of the times and Ensure that If you have completed all your tasks, make sure to use the "finish" command and args also should blank. do not use this command until each and every command completed successfully\n' + + '\n' + + 'GOALS:\n' + + '\n' + + goalsStr + + '\n' + + `Key Points: + - Always include the "args" object with the necessary arguments for the command. This field is required and must not be omitted. + - Start directly with Playwright actions as described in the user task, without adding extraneous steps or assertions. + - Include assertions like 'expect' statements or wait functions such as 'waitForLoadState' only when they are specifically requested in the user task. + - Minimal, relevant comments should be used to clarify complex actions or essential aspects of the test's purpose. + - Apply 'frameLocator' for content in nested iframes, as needed based on the task requirements.\n + - Don't log the output always return that + `+ + 'Constraints:\n' + + '1. ~4000 word limit for short term memory. \n' + + '2. If you are unsure how you previously did something or want to recall past events, thinking about similar events will help you remember.\n' + + '3. No user assistance\n' + + '4. Exclusively use the commands listed below e.g. command_name\n' + + '\n' + + 'Commands:\n' + + `// for navigating to the page + - await page.goto('https://github.com/login'); + // for clicking on the button + - await page.getByRole('button', { name: 'Submit' }).click(); + // for filling the input fields + - await page.getByLabel('Username or email address').fill('username'); + - await page.getByLabel('Password').fill('password'); + // Text input + - await page.getByRole('textbox').fill('Peter'); + // Date input + - await page.getByLabel('Birth date').fill('2020-02-02'); + // Time input + - await page.getByLabel('Appointment time').fill('13:15'); + // Local datetime input + - await page.getByLabel('Local time').fill('2020-03-02T05:15'); + - await page.locator('[data-test="password"]').fill('secret_sauce'); + - await page.getByRole('button', { name: 'Sign in' }).click(); + - await page.innerText('html') + - page.getByRole('listitem').filter({ hasText: 'Product 2' }); + - await page.getByRole('listitem').filter({ hasText: 'Product 2' }).getByRole('button', { name: 'Add to cart' }).click(); + - page.locator('button.buttonIcon.episode-actions-later'); + - await expect(page.getByText('welcome')).toBeVisible(); + - await expect(page.getByText('welcome')).toBeVisible(); + - await page.innerText(selector); + - await page.innerText(selector, options); + - const page = await browser.newPage(); + - await page.goto('https://keycode.info'); + - await page.press('body', 'A'); + - await page.screenshot({ path: 'A.png' }); + - await page.press('body', 'ArrowLeft'); + - await page.screenshot({ path: 'ArrowLeft.png' }); + - await page.press('body', 'Shift+O'); + - await page.screenshot({ path: 'O.png' }); + - await browser.close(); + // click on any links + - await page.click('a[href="https://blog.sbensu.com/posts/demand-for-visual-programming/"]');\n\n`+ + '\n' + + '\n' + + 'Resources:\n' + + '1. Internet access for searches and information gathering.\n' + + '2. Long Term memory management.\n' + + '3. GPT-3.5 powered Agents for delegation of simple tasks.\n' + + '\n' + + 'Performance Evaluation:\n' + + '1. Continuously review and analyze your actions to ensure you are performing to the best of your abilities.\n' + + '2. Constructively self-criticize your big-picture behavior constantly.\n' + + '3. Reflect on past decisions and strategies to refine your approach.\n' + + '4. Every command has a cost, so be smart and efficient. Aim to complete tasks in the least number of steps.' + }]; +} + +export async function findInPage(page: Page, task: string): Promise { + + const [currentPageUrl, currentPageTitle, siteOverview] = await Promise.all([ + await page.evaluate('location.href'), + await page.evaluate('document.title'), + parseSite(page).then((html) => removeBlankTags(html).slice(0, 24000)), + ]).catch((error: any) => { + console.log(error.message + " " + error.stack); + return error.message + " " + error.stack; + }); + + return ` + You are a programmer and your job is to pick out information in code to a pm. You are working on an html file. You will extract the necessary content asked from the information provided. + + Context: + Your computer is a mac. Cmd is the meta key, META. + The browser is already open. + Current page url is ${currentPageUrl}. + Current page title is ${currentPageTitle}. + + Here is the overview of the site. Format is in html: + \`\`\` + ${siteOverview} + \`\`\` + \n\n + + User Task: ${task} + + `; +} + +export const mainResponseFormat = [ + { + name: "response", + description: "Generate a detailed response based on provided details.", + parameters: { + type: "object", + properties: { + thoughts: { + type: "object", + description: "A collection of thoughts including reasoning and plans.", + properties: { + text: { + type: "string", + description: "thought" + }, + reasoning: { + type: "string", + description: "reasoning" + }, + plan: { + type: "string", + description: "- short bulleted\n- list that conveys\n- long-term plan" + }, + criticism: { + type: "string", + description: "constructive self-criticism" + }, + speak: { + type: "string", + description: "thoughts summary to say to user" + } + }, + required: ["text", "reasoning", "plan", "criticism", "speak"] + }, + command: { + type: "object", + description: "A command to be executed based on the current context.", + properties: { + name: { + type: "string", + description: "The name of the command to execute." + }, + args: { + type: "object", + description: "Arguments required for executing the command.", + "additionalProperties": true, + } + }, + required: ["name", "args"] + } + }, + required: ["thoughts", "command"] + } + } +]; + +export const taskArrResponseFormat = [ + { + name: "taskListResponse", + description: "Generate a response containing a list of tasks.", + parameters: { + type: "object", + properties: { + tasks: { + type: "array", + description: "A list of tasks to be performed, represented as strings.", + items: { + type: "string", + description: "A single task description." + } + } + }, + required: ["tasks"] // Ensure that the 'tasks' array is provided + } + } +]; + +export const playwrightCodeResponseFormat = [ + { + name: "playwrightCode", + description: "Generates playwright Code code for a given task", + parameters: { + type: "object", + properties: { + code: { + type: "string", + description: "Playwright code for a single task." + } + }, + required: ["code"] + } + } +]; + +export async function execPlayWrightCode(page: Page, code: string): Promise { + const dependencies = [ + { param: 'page', value: page }, + { param: 'expect', value: expect }, + ]; + + const func = AsyncFunction(...dependencies.map((d) => d.param), code); + const args = dependencies.map((d) => d.value); + return await func(...args); +} + +export async function goToLink(page: Page, url: string): Promise { + await page.goto(url); +} \ No newline at end of file diff --git a/JS/edgechains/arakoodev/src/scraper/src/lib/playwright.ts b/JS/edgechains/arakoodev/src/scraper/src/lib/playwright.ts index 56e471c4..27516bd8 100644 --- a/JS/edgechains/arakoodev/src/scraper/src/lib/playwright.ts +++ b/JS/edgechains/arakoodev/src/scraper/src/lib/playwright.ts @@ -1,200 +1,175 @@ import { chromium, Page } from "playwright"; -import { expect } from "@playwright/test"; import axios from "axios"; -import { parseArr, parseSite, preprocessJsonInput } from "../utils/index"; import { retry } from "@lifeomic/attempt"; -import { removeBlankTags } from "../utils/page-parser"; +import { + playwrightCodeResponseFormat, + findInPage, + generatePrompt, + generateTaskArrPrompt, + handleHistory, + mainResponseFormat, + taskArrResponseFormat, + execPlayWrightCode, + goToLink +} from "../actions"; +const openaiUrl = "https://api.openai.com/v1/chat/completions"; + +type messages = { role: string, content: string }[] export class Playwright { apiKey: string; - - constructor({ apiKey }: { apiKey: string }) { + model: string; + verbose: boolean; + constructor({ apiKey, model, verbose = true }: { apiKey: string, model: string, verbose?: boolean }) { this.apiKey = apiKey; + this.model = model; + this.verbose = verbose; } - async #createPrompt({ task, page, completedTaskArr }) { - const [currentPageUrl, currentPageTitle, siteOverview] = await Promise.all([ - page.evaluate("location.href"), - page.evaluate("document.title"), - parseSite(page).then((html) => removeBlankTags(html).slice(0, 20000)), - ]); + static #history: messages = []; - const completedActions = completedTaskArr || []; + #getHistory() { + return Playwright.#history + } - return ` - You are a Senior SDET tasked with writing Playwright code for testing purposes. Your role involves implementing specific task-based code segments within a larger test file, following the instructions provided closely. Assume that common imports like 'test' and 'expect' from '@playwright/test' are already at the top of the file. - - Context: - - Your computer is a Mac. Cmd is the meta key, META. - - The browser is already open. - - Current page URL: ${currentPageUrl}. - - Current page title: ${currentPageTitle}. - - [NO NEED TO WRITE CODE TO OPEN THIS URL AGAIN ${currentPageUrl}, THE BROWSER IS ALREADY OPEN] - - HumanMessage Write Playwright code for this: ${task} - - Overview of the site in HTML format: - \\\ - ${siteOverview} - \\\ - - Completed Actions: ${completedActions.join(", ")} - - Key Points: - - Don't navigate to a new page unless explicitly instructed. - - Don't give every links like this await page.click('a[href="https:^]'), only click on the specific link mentioned in the task. - - Please note that if there is a timeout error, you may need to increase the timeout value or use a different method to locate the input fields. - - Follow the following Playwright actions for the task: - // for navigating to the page - - await page.goto('https://github.com/login'); - // for clicking on the button - - await page.getByRole('button', { name: 'Submit' }).click(); - // for filling the input fields - - await page.getByLabel('Username or email address').fill('username'); - - await page.getByLabel('Password').fill('password'); - // Text input - - await page.getByRole('textbox').fill('Peter'); - // Date input - - await page.getByLabel('Birth date').fill('2020-02-02'); - // Time input - - await page.getByLabel('Appointment time').fill('13:15'); - // Local datetime input - - await page.getByLabel('Local time').fill('2020-03-02T05:15'); - - await page.locator('[data-test="password"]').fill('secret_sauce'); - - await page.getByRole('button', { name: 'Sign in' }).click(); - - await page.innerText('html') - - page.getByRole('listitem').filter({ hasText: 'Product 2' }); - - await page.getByRole('listitem').filter({ hasText: 'Product 2' }).getByRole('button', { name: 'Add to cart' }).click(); - - page.locator('button.buttonIcon.episode-actions-later'); - - await expect(page.getByText('welcome')).toBeVisible(); - - await expect(page.getByText('welcome')).toBeVisible(); - - await page.innerText(selector); - - await page.innerText(selector, options); - - const page = await browser.newPage(); - - await page.goto('https://keycode.info'); - - await page.press('body', 'A'); - - await page.screenshot({ path: 'A.png' }); - - await page.press('body', 'ArrowLeft'); - - await page.screenshot({ path: 'ArrowLeft.png' }); - - await page.press('body', 'Shift+O'); - - await page.screenshot({ path: 'O.png' }); - - await browser.close(); - // click on any links - - await page.click('a[href="https://blog.sbensu.com/posts/demand-for-visual-programming/"]'); - - - Start directly with Playwright actions without adding extraneous steps or assertions. - - Include assertions like 'expect' statements or wait functions such as 'waitForLoadState' only when they are specifically requested in the user task. - - Apply 'frameLocator' for content in nested iframes, as needed based on the task requirements. - - Expected Code Format: - \\\ - // Insert Playwright code based on the task description. Begin with necessary actions directly, and include 'waitForLoadState', assertions, or 'expect' statements only if explicitly requested in the task. Comments should be concise and pertinent, especially for complex actions or decisions.] - \\\ - - The objective is to create Playwright code that is efficient, precise, and perfectly aligned with the task's requirements, integrating seamlessly into the larger test file. All actions should be relevant and necessary, catering to a senior-level professional's understanding of the testing scenario. - - - Examples: - go to hacker news - await page.goto('https://news.ycombinator.com/') - click on the first link - page.click('a[href="https://blog.sbensu.com/posts/demand-for-visual-programming/"]') - give me all the text of this page - await page.innerText('html'); - `; + #setHistory(message: { role: string, content: string }) { + Playwright.#history.push(message) } - #createPromptForTaskArr(task: string) { - return ` - Given the following task description: + async #getThoughtAndCommands() { + const openaiResponse = await this.#openAIRequest([ + ...this.#getHistory(), + { + role: "user", + content: 'Determine which next command to use, ensuring that the "args" field is always populated with the necessary arguments, and respond using the format specified above.' + } + ], mainResponseFormat, mainResponseFormat[0].name); + this.#setHistory({ role: "assistant", content: openaiResponse.function_call.arguments }); + return await JSON.parse(openaiResponse.function_call.arguments); + } - ${task} + async #interactWithPage(page: Page, task: string) { + const taskMessage = [{ + role: "user", + content: generateTaskArrPrompt(task) + }] + + const taskResponse = await this.#openAIRequest(taskMessage, taskArrResponseFormat, taskArrResponseFormat[0].name) + const taskArr = JSON.parse(taskResponse.function_call.arguments).tasks + if (this.verbose == true) { console.log("GOALS: ", taskArr) }; + const history = handleHistory(taskArr); + history.map((element) => { this.#setHistory(element) }); + let commandRunningAttempt: number = 0; + let playwrightResponse: string = ""; + let isFinishCount: number = 0; + while (1) { + + const parsedOpenaiResponse = await this.#getThoughtAndCommands(); + let command = parsedOpenaiResponse.command; + if (this.verbose == true) { console.log(parsedOpenaiResponse) } + + if (command.name == "finish") { + isFinishCount++; + if (isFinishCount >= 2) { + break; + } + this.#setHistory({ role: "assistant", content: `Command finish args: ${command?.args || "NO ARGUMENTS PROVIDED"}` }); + this.#setHistory({ role: "system", content: `I need to insure that all the commmands is finished or Not?` }); + continue; + } - Extract the key actions from this task and return them as an array of strings. Each action should be a separate string in the array. If the task description contains syntax errors or you think a command can be improved for better clarity and effectiveness, please make the necessary corrections and improvements. For example: + if (command.name == "goToLink") { + try { + await goToLink(page, command.args.url); + this.#setHistory({ role: "system", content: `Command ${command.name} returned: Success` }); + } catch (error: any) { + this.#setHistory({ role: "system", content: `Args: ${command.args}. Error: ${error.message + error.stack}` }); + } + continue; + } - Input: - "Go to Hacker News and click on the first link. Then give me all the text of this page." + if (command.name == "findInPage" || commandRunningAttempt >= 3) { + const findInPagePrompt = await findInPage(page, JSON.stringify(parsedOpenaiResponse)); + const findInPageRespones = await this.#openAIRequest([ + { role: "system", content: findInPagePrompt }, + { role: "user", content: "Try finding something in the html body content of the current webpage. use this content to figure out your next step. The task should be written question explaining what you want to find. You can find something like css selector, classes, link, tag, etc. So we can locate the element" } + ]); + if (this.verbose == true) { console.log(findInPageRespones.content); } + this.#setHistory({ role: "assistant", content: findInPageRespones.content }); + command = await this.#getThoughtAndCommands(); + } - Output: - \`\`\` - [ - "Navigate to the Hacker News website by entering the URL 'https://news.ycombinator.com/' in the browser", - "Identify and click on the first link displayed on the Hacker News homepage", - "Extract and return all the text content from the page" - ] - \`\`\` + const prompt = await generatePrompt(page, JSON.stringify(command)); + const playwrightCodeResponse = await this.#openAIRequest([ + { role: "user", content: prompt } + ], playwrightCodeResponseFormat, playwrightCodeResponseFormat[0].name); + const playwrightCode = JSON.parse(playwrightCodeResponse.function_call.arguments).code; + if (this.verbose == true) { console.log("Code To Run: ", playwrightCode); } + this.#setHistory({ role: "assistant", content: playwrightCode }); + + let result: string = ''; + try { + playwrightResponse = await execPlayWrightCode(page, playwrightCode); + result = `Command ${command.name} returned: Success`; + this.#setHistory({ role: "system", content: result }); + commandRunningAttempt = 0; + } catch (error: any) { + if (command.name === "ERROR") { + result = `Error: ${command.args}.`; + this.#setHistory({ role: "system", content: result }); + continue; + } - Input: - "Go to random website. we have fields in this pattern First Name Last Name Company Name Role in Company Address Email Phone Number and then fill this fields John Smith IT Solutions Analyst 98 North Road " + result = `Unknown command '${command.name}'. Please refer to the 'COMMANDS' list for available commands and only respond in the specified JSON format. Assistant Reply commands: ${command}\nResult: ${error.message} \n ${error.stack}`; - Output: - \`\`\` - [ - "Navigate to the random website by entering the URL 'https://www.randomwebsite.com/' in the browser", - "Fill the form fields with the following data: First Name: John, Last Name: Smith, Company Name: IT Solutions, Role in Company: Analyst, Address: 98 North Road, Email:", - ] - \`\`\` + this.#setHistory({ role: "system", content: result }); + commandRunningAttempt++; + } + } - Input: - "Go to google and search for the term 'automation'. Click on the first link and extract the text from the page." + return playwrightResponse; + } - Output: - \`\`\` - [ - "Navigate to the random website by entering the URL 'https://google.com' in the browser", - "Search for the term 'automation' in the search bar and hit Enter key", - "Click on the first link displayed in the search results", - "Extract and return all the text content from the page" - ] - \`\`\` + async #openAIRequest( + message: { role: string, content: string }[], + functions?: { + name: string, + description: string, + parameters: {} + }[], + functionCall?: string + ): Promise { + const funsObj = functions ? { + functions, + function_call: functionCall ? { name: functionCall } : "auto", + } : {}; - Ensure that each action is specific, clear, and comprehensive to facilitate precise implementation. - `; - } - async #openAIRequest({ prompt }: { prompt: string }) { - return await retry(async () => { - const response = await axios.post( - "https://api.openai.com/v1/chat/completions", + try { + + const response = await axios.post(openaiUrl, { - model: "gpt-3.5-turbo-16k", - messages: [{ role: "user", content: prompt }], - max_tokens: 1000, + model: this.model, + messages: message, + max_tokens: 1024, temperature: 0.7, + ...funsObj }, { headers: { - Authorization: `Bearer ${this.apiKey}`, + Authorization: `Bearer ${this.apiKey} `, "content-type": "application/json", }, } ); - return response.data.choices[0].message.content; - }); - } - - async #findInPage({ page, task }): Promise { - const prompt = ` - You are a programmer and your job is to pick out information in code to a pm. You are working on an html file. You will extract the necessary content asked from the information provided. - - Context: - Your computer is a mac. Cmd is the meta key, META. - The browser is already open. - Current page url is ${await page.evaluate("location.href")}. - Current page title is ${await page.evaluate("document.title")}. - Humman message: ${task} - - Here is the overview of the site. Format is in html: - \`\`\` - ${removeBlankTags(await parseSite(page)).slice(0, 25000)} - \`\`\` - `; - - return await this.#openAIRequest({ prompt }); - } - - async #execPlayWrightCode({ page, code }: { page: Page; code: string }) { - const AsyncFunction = async function () {}.constructor; - const dependencies = [{ param: "page", value: page }]; - const func = AsyncFunction(...dependencies.map((d) => d.param), code); - const args = dependencies.map((d) => d.value); - return await func(...args); + return response.data.choices[0].message; + } catch (error: any) { + console.log( + error.message + + "\n" + error.stack + ) + } } /** @@ -203,56 +178,14 @@ export class Playwright { * @param url - URL to navigate to default is https://www.google.com * @param headless - Run in headless mode default is false **/ - async call({ task, url, headless = true }: { task: string; url?: string; headless?: boolean }) { + async call({ task, url, headless = true }: { task: string, url?: string, headless?: boolean }) { + if (!this.apiKey || this.apiKey == undefined) { + console.log("Please provide apiKey, You can find you apikey here https://platform.openai.com/") + process.exit(); + } const browser = await chromium.launch({ headless }); const page = await browser.newPage(); await page.goto(url || "https://www.google.com"); - - const taskPrompt = this.#createPromptForTaskArr(task); - const taskArr: any = parseArr(await this.#openAIRequest({ prompt: taskPrompt })); - - const completedTaskArr: string[] = []; - let response: string = ""; - let err; - - for (const task of taskArr) { - if (response) break; - - console.log(task); - let success = false; - let action = ""; - const prompt = await this.#createPrompt({ task, page, completedTaskArr }); - let errExecIndex = 0; - - while (!success) { - let res: any = preprocessJsonInput( - await this.#openAIRequest({ prompt: prompt + err }) - ); - - if (errExecIndex > 3) { - const findInResponse = await this.#findInPage({ page, task }); - const retryPrompt = await this.#createPrompt({ - task: `We are getting this error three times: ${err}. Try writing Playwright code using this: ${findInResponse}`, - page, - completedTaskArr, - }); - res = preprocessJsonInput(await this.#openAIRequest({ prompt: retryPrompt })); - } - - try { - const finalResponse = await this.#execPlayWrightCode({ page, code: res }); - if (finalResponse) response = finalResponse; - completedTaskArr.push(action); - success = true; - } catch (error: any) { - err = `\n\nError in this command ${action} Error: ${error.message}\n${error.stack} Try another way to do this action`; - console.log("Error: ", error.message); - errExecIndex++; - } - } - } - - await browser.close(); - return response; + return await this.#interactWithPage(page, task); } -} +} \ No newline at end of file diff --git a/JS/edgechains/arakoodev/src/scraper/src/utils/index.ts b/JS/edgechains/arakoodev/src/scraper/src/utils/index.ts index 9310d32a..5905e20d 100644 --- a/JS/edgechains/arakoodev/src/scraper/src/utils/index.ts +++ b/JS/edgechains/arakoodev/src/scraper/src/utils/index.ts @@ -1,5 +1,6 @@ const codeRegex = /```(.*)(\r\n|\r|\n)(?[\w\W\n]+)(\r\n|\r|\n)```/; const codeRegex2 = /```javascript(.*)(\r\n|\r|\n)(?[\w\W\n]+)(\r\n|\r|\n)```/i; +const codeRegex3 = /```json(.*)(\r\n|\r|\n)(?[\w\W\n]+)(\r\n|\r|\n)```/i; export function preprocessJsonInput(text) { try { @@ -14,10 +15,16 @@ export function parseArr(text) { if (text.startsWith("[") && text.endsWith("]")) { return JSON.parse(text); } - if (text.startsWith("```Javascript") || text.startsWith("```javascript")) { + if (text.startsWith("```Javascript")) { return text.match(codeRegex2).groups.code.trim(); } - return text.match(codeRegex).groups.code.trim(); + if (text.startsWith("```json")) { + return text.match(codeRegex3).groups.code.trim(); + } + if (text.startsWith("```")) { + return text.match(codeRegex).groups.code.trim(); + } + return text; } catch (e) { console.log({ text }); throw new Error("No code found"); diff --git a/JS/edgechains/examples/rpa-challenge/jsonnet/main.jsonnet b/JS/edgechains/examples/rpa-challenge/jsonnet/main.jsonnet index 99c9bcfc..c4b2258f 100644 --- a/JS/edgechains/examples/rpa-challenge/jsonnet/main.jsonnet +++ b/JS/edgechains/examples/rpa-challenge/jsonnet/main.jsonnet @@ -1,8 +1,71 @@ local promptTemplate = ||| - go to https://rpachallenge.com/ and click on the start button we have fiels in this pattern First Name Last Name Company Name Role in Company Address Email Phone Number and then fill this fields John Smith IT Solutions Analyst 98 North Road jsmith@itsolutions.co.uk 40716543298 in the given inputs and click the input with value Submit and then fill this fields Jane Dorsey MediCare Medical Engineer 11 Crown Street jdorsey@mc.com 40791345621 in the given inputs and click the input with value Submit and then fill this fields Albert Kipling Waterfront Accountant 22 Guild Street kipling@waterfront.com 40735416854 in the given inputs and click the input with value Submit and then fill this fields Michael Robertson MediCare IT Specialist 17 Farburn Terrace mrobertson@mc.com 40733652145 in the given inputs and click the input with value Submit and then fill this fields Doug Derrick Timepath Inc. Analyst 99 Shire Oak Road dderrick@timepath.co.uk 40799885412 in the given inputs and click the input with value Submit and then fill this fields Jessie Marlowe Aperture Inc. Scientist 27 Cheshire Street jmarlowe@aperture.us40733154268 in the given inputs and click the input with value Submit and then fill this fields Stan Hamm Sugarwell Advisor 10 Dam Road shamm@sugarwell.org 40712462257 in the given inputs and click the input with value Submit and then fill this fields Michelle Norton Aperture Inc. Scientist 13 White Rabbit Street mnorton@aperture.us 40731254562 in the given inputs and click the input with value Submit and then fill this fields Stacy Shelby TechDev HR Manager 19 Pineapple Boulevard sshelby@techdev.com 40741785214 in the given inputs and click the input with value Submit and then fill this fields Lara Palmer Timepath Inc. Programmer 87 Orange Street lpalmer@timepath.co.uk 40731653845 in the given inputs and click the input with value Submit + Given the following task description: + {task} + Extract the key actions from this task and return them as an array of strings. Each action should be a separate string in the array. If the task description contains syntax errors or you think a command can be improved for better clarity and effectiveness, please make the necessary corrections and improvements. For example: + Using the following list of objects, generate a series of instructions to fill out a form with the specified data. After each entry, include a command to submit the form. Follow this structure for each object: + Extract the details from each object in the list. + Format the details as follows: "Fill the form fields with the following data: First Name: [First Name], Last Name: [Last Name], Company Name: [Company Name], Role in Company: [Role in Company], Address: [Address], Email: [Email], Phone Number: [Phone Number]". + After each formatted string, append the text: "Identify and click on the input with the value 'Submit'". + + Examples: + + Input: + "Go to Hacker News and click on the first link. Then give me all the text of this page." + Response Format: + \`\`\` + {[ + "Navigate to the Hacker News website by entering the URL 'https://news.ycombinator.com/' in the browser", + "Identify and click on the first link displayed on the Hacker News homepage", + "Extract all the text from the page", + "Return that containt using return statement" + ]} + \`\`\` + + Input: + "Go to google and search for the term 'automation'. Click on the first link and extract the text from the page." + + Response Format: + \`\`\` + {[ + "Navigate to the random website by entering the URL 'https://google.com' in the browser", + "Search for the term 'automation' in the search bar and hit Enter key", + "Click on the first link displayed in the search results", + "Extract all the text from the page", + "Return that containt using return statement" + ]} + \`\`\` + \n + Ensure that each action is specific, clear, and comprehensive to facilitate precise implementation. |||; + +local openAIResponseFormat = [ + { + name: "taskListResponse", + description: "Generate a response containing a list of tasks.", + parameters: { + type: "object", + properties: { + tasks: { + type: "array", + description: "A list of tasks to be performed, represented as strings.", + items: { + type: "string", + description: "A single task description." + } + } + }, + required: ["tasks"] // Ensure that the 'tasks' array is provided + } + } +]; + +local excelUrl = "https://rpachallenge.com/assets/downloadFiles/challenge.xlsx"; local key = std.extVar('openai_api_key'); -local content = arakoo.native("call")({task:promptTemplate, openai:key}); -content \ No newline at end of file +local excelData = arakoo.native('downloadExcelData')({url:excelUrl}); +local taskString = "go to https://rpachallenge.com/ click on the start button" + excelData; +local updatedPrompt = std.strReplace(promptTemplate, '{task}',taskString + "\n"); +local mainPrompt = arakoo.native("openAICall")({prompt:updatedPrompt, functions:openAIResponseFormat, openAIKey:key}); +local doMainTasks = arakoo.native("doMainTasks")({task:mainPrompt, openai:key}); +doMainTasks \ No newline at end of file diff --git a/JS/edgechains/examples/rpa-challenge/jsonnet/secrets.jsonnet b/JS/edgechains/examples/rpa-challenge/jsonnet/secrets.jsonnet index 0f65557f..fe716556 100644 --- a/JS/edgechains/examples/rpa-challenge/jsonnet/secrets.jsonnet +++ b/JS/edgechains/examples/rpa-challenge/jsonnet/secrets.jsonnet @@ -1,4 +1,3 @@ - local OPENAI_API_KEY = "sk-***"; { diff --git a/JS/edgechains/examples/rpa-challenge/package.json b/JS/edgechains/examples/rpa-challenge/package.json index 0403b144..c69ba738 100644 --- a/JS/edgechains/examples/rpa-challenge/package.json +++ b/JS/edgechains/examples/rpa-challenge/package.json @@ -14,8 +14,10 @@ "@arakoodev/edgechains.js": "file:../../arakoodev", "@arakoodev/jsonnet": "^0.25.0", "file-uri-to-path": "^2.0.0", + "filereader": "^0.10.3", "path": "^0.12.7", "sync-rpc": "^1.3.6", + "xlsx": "^0.18.5", "zod": "^3.23.8" }, "devDependencies": { diff --git a/JS/edgechains/examples/rpa-challenge/src/index.ts b/JS/edgechains/examples/rpa-challenge/src/index.ts index 57084fba..32a39c4d 100644 --- a/JS/edgechains/examples/rpa-challenge/src/index.ts +++ b/JS/edgechains/examples/rpa-challenge/src/index.ts @@ -10,14 +10,18 @@ const app = server.createApp(); const jsonnet = new Jsonnet(); const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const call = createClient(path.join(__dirname, "./lib/playwright.cjs")); +const doMainTasks = createClient(path.join(__dirname, "./lib/playwright.cjs")); +const openAICall = createClient(path.join(__dirname, "./lib/generateTask.cjs")); +const downloadExcelData = createClient(path.join(__dirname, "./lib/downloadExcelTask.cjs")); + app.get("/", async (c: any) => { const key = JSON.parse( jsonnet.evaluateFile(path.join(__dirname, "../jsonnet/secrets.jsonnet")) ).openai_api_key; - jsonnet.extString("openai_api_key", key); - jsonnet.javascriptCallback("call", call); + jsonnet.javascriptCallback("downloadExcelData", downloadExcelData); + jsonnet.javascriptCallback("doMainTasks", doMainTasks); + jsonnet.javascriptCallback("openAICall", openAICall); let response = jsonnet.evaluateFile(path.join(__dirname, "../jsonnet/main.jsonnet")); return c.text(response); }); diff --git a/JS/edgechains/examples/rpa-challenge/src/lib/downloadExcelTask.cts b/JS/edgechains/examples/rpa-challenge/src/lib/downloadExcelTask.cts new file mode 100644 index 00000000..b202d12b --- /dev/null +++ b/JS/edgechains/examples/rpa-challenge/src/lib/downloadExcelTask.cts @@ -0,0 +1,18 @@ +import axios from "axios"; +import XLSX from "xlsx"; + +async function downloadExcelData({ url }: { url: string }) { + try { + let axiosResponse = await axios(url, { responseType: "arraybuffer" }); + const workbook = XLSX.read(axiosResponse.data); + + let worksheets = workbook.SheetNames.map((sheetName: any) => { + return { sheetName, data: XLSX.utils.sheet_to_json(workbook.Sheets[sheetName]) }; + }); + return JSON.stringify(worksheets[0].data); + } catch (error) { + console.log(error); + } +} + +module.exports = downloadExcelData; diff --git a/JS/edgechains/examples/rpa-challenge/src/lib/generateTask.cts b/JS/edgechains/examples/rpa-challenge/src/lib/generateTask.cts new file mode 100644 index 00000000..78a58e34 --- /dev/null +++ b/JS/edgechains/examples/rpa-challenge/src/lib/generateTask.cts @@ -0,0 +1,25 @@ +const { OpenAI } = require("@arakoodev/edgechains.js/ai"); + +function openAICall({ prompt, functions, openAIKey }: { prompt: string, functions: any, openAIKey: string }) { + try { + const openai = new OpenAI({ apiKey: openAIKey }); + const completion = openai + .chatWithFunction({ + model: "gpt-3.5-turbo-0613", + messages: [{ role: "user", content: prompt }], + functions, + function_call: { name: functions[0].name }, + }) + .then((completion: any) => { + return JSON.stringify(JSON.parse(completion.function_call.arguments).tasks); + }) + .catch((error: any) => { + console.error(error); + }); + return completion; + } catch (error) { + console.error(error); + } +}; + +module.exports = openAICall; \ No newline at end of file diff --git a/JS/edgechains/examples/rpa-challenge/src/lib/playwright.cts b/JS/edgechains/examples/rpa-challenge/src/lib/playwright.cts new file mode 100644 index 00000000..a9eb0711 --- /dev/null +++ b/JS/edgechains/examples/rpa-challenge/src/lib/playwright.cts @@ -0,0 +1,12 @@ +import { Playwright } from "@arakoodev/edgechains.js/scraper"; + +async function doMainTasks({ task, openai }: { task: string; openai: string }) { + const scraper = new Playwright({ apiKey: openai, model: "gpt-4-1106-preview" }); + try { + return JSON.stringify(await scraper.call({ task, headless: false })) + } catch (error) { + console.log(error); + } +} + +module.exports = doMainTasks; diff --git a/JS/edgechains/examples/rpa-challenge/src/lib/playwright.ts b/JS/edgechains/examples/rpa-challenge/src/lib/playwright.ts deleted file mode 100644 index 70fc3026..00000000 --- a/JS/edgechains/examples/rpa-challenge/src/lib/playwright.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Playwright } from "@arakoodev/edgechains.js/scraper"; - -async function call({ task, openai }: { task: string; openai: string }) { - const scraper = new Playwright({ apiKey: openai }); - try { - return await scraper.call({ task, headless: false }); - } catch (error) { - console.log(error); - } -} - -module.exports = call; diff --git a/package.json b/package.json index d59631d6..9ddd1f69 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,8 @@ { "dependencies": { "@microsoft/eslint-formatter-sarif": "^3.1.0", + "@playwright/test": "^1.46.0", + "axios": "^1.7.3", "@typescript-eslint/eslint-plugin": "^8.1.0", "@typescript-eslint/parser": "^8.1.0", "eslint": "^8.57.0",