Merge with upstream/dev

Author: ruhanaazam

README.md

@@ -11,7 +11,7 @@ This provides a natural language way to interacting with a web browser:
 - Manage and automate tasks on project management platforms (like JIRA) by filtering issues, easing the workflow for users.
 - Provide personal shopping assistance, suggesting products based on the user's needs, such as storage options for game cards.
 
-While Agent-E is growing, it is already equipped to handle a versatile range of tasks, but the best task is the one that you come up with. So, take it for a spin and tell us what you were able to do with it. For more information see our [blog article](https://blog.emergence.ai/2024/03/28/distilling-the-web-agent.html).
+While Agent-E is growing, it is already equipped to handle a versatile range of tasks, but the best task is the one that you come up with. So, take it for a spin and tell us what you were able to do with it. For more information see our [blog article](https://www.emergence.ai/blog/distilling-the-web-for-multi-agent-automation).
 
 
 ## Quick Start
@@ -30,7 +30,8 @@ While Agent-E is growing, it is already equipped to handle a versatile range of
 - .env file in project root is needed with the following (sample `.env-example` is included for convience):
     - Follow the directions in the sample file
     - You will need to set `AUTOGEN_MODEL_NAME` (for example `gpt-4-turbo-preview`) and `AUTOGEN_MODEL_API_KEY`
-    - If you are using a model other than OpenAI, you need to set `AUTOGEN_MODEL_BASE_URL` for example `https://api.groq.com/openai/v1`
+    - If you are using a model other than OpenAI, you need to set `AUTOGEN_MODEL_BASE_URL` for example `https://api.groq.com/openai/v1` or `https://<REPLACE_AI_SERVICES>.openai.azure.com` on [Azure](https://azure.microsoft.com/).
+    - For [Azure](https://azure.microsoft.com/), you'll also need to configure `AUTOGEN_MODEL_API_TYPE=azure` and `AUTOGEN_MODEL_API_VERSION` (for example `2023-03-15-preview`) variables.
     - If you want to use local chrome browser over playwright browser, go to chrome://version/ in chrome, find the path to your profile and set `BROWSER_STORAGE_DIR` to the path value
 
 ### pip issues
@@ -155,6 +156,39 @@ html_theme = 'sphinx_rtd_theme'
 7. Build the documentation, from `docs` directory, run: `sphinx-build -b html . _build`
 
 
+## Open-source models
+
+Using open-source models is possible through LiteLLM with Ollama. Ollama allows users to run language models locally on their machines, and LiteLLM translates OpenAI-format inputs to local models' endpoints. To use open-source models as Agent-E backbone, follow the steps below:
+
+1. Install LiteLLM
+    ```bash
+    pip install 'litellm[proxy]'
+    ```
+2. Install Ollama
+    * For Mac and Windows, download [Ollama](https://ollama.com/download).
+    * For Linux:
+        ```bash
+        curl -fsSL https://ollama.com/install.sh | sh
+        ```
+3. Pull Ollama models
+    Before you can use a model, you need to download it from the library. The list of available models is [here](https://ollama.com/library). Here, we use Mistral v0.3:
+    ```bash
+    ollama pull mistral:v0.3
+    ```
+4. Run LiteLLM
+    To run the downloaded model with LiteLLM as a proxy, run:
+    ```bash
+    litellm --model ollama_chat/mistral:v0.3
+    ```
+5. Configure model in Autogen
+    Configure the `.env` file as follows. Note that the model name and API keys are not needed since the local model is already running.
+    ```bash
+    AUTOGEN_MODEL_NAME=NotRequired
+    AUTOGEN_MODEL_API_KEY=NotRequired
+    AUTOGEN_MODEL_BASE_URL=http://0.0.0.0:400
+    ```
+
+
 ## TODO
 
 - Action verification - Responding from every skill with changes that took place in the DOM (Mutation Observers) so that the LLM can judge whether the skill did execute properly or not
@@ -173,7 +207,7 @@ html_theme = 'sphinx_rtd_theme'
 
 ## Social:
 
-[Discord](https://discord.gg/czHhPDMk)
+[Discord](https://discord.gg/wgNfmFuqJF)
 
 ## Contributing
 

ae/__init__.py

@@ -1 +1 @@
-from ae import core
+from ae import core  # type: ignore # noqa: F401

ae/config.py

@@ -23,4 +23,4 @@
 
 if not os.path.exists(PROJECT_TEMP_PATH):
     os.makedirs(PROJECT_TEMP_PATH)
-    print(f"Created temp folder at: {PROJECT_TEMP_PATH}")
+    print(f"Created temp folder at: {PROJECT_TEMP_PATH}")

ae/core/__init__.py

@@ -1,10 +1,8 @@
 from ae.core import agents
 from ae.core import memory
 from ae.core import skills
-
 from ae.core.autogen_wrapper import AutogenWrapper
 from ae.core.playwright_manager import PlaywrightManager
-from ae.core.post_process_responses import final_reply_callback_browser_agent
 from ae.core.post_process_responses import final_reply_callback_user_proxy
 from ae.core.prompts import LLM_PROMPTS
 from ae.core.system_orchestrator import SystemOrchestrator

ae/core/agents/browser_nav_agent.py

@@ -2,25 +2,23 @@
 from string import Template
 
 import autogen  # type: ignore
-from autogen.agentchat.conversable_agent import register_function  # type: ignore
 
 from ae.core.memory.static_ltm import get_user_ltm
-from ae.core.post_process_responses import final_reply_callback_browser_agent as print_message_from_user_proxy  # type: ignore
-from ae.core.post_process_responses import final_reply_callback_user_proxy as print_message_from_browser_agent  # type: ignore
 from ae.core.prompts import LLM_PROMPTS
 from ae.core.skills.click_using_selector import click as click_element
+
+# from ae.core.skills.enter_text_and_click import enter_text_and_click
 from ae.core.skills.enter_text_using_selector import bulk_enter_text
-from ae.core.skills.enter_text_and_click import enter_text_and_click
-from ae.core.skills.pdf_text_extractor import extract_text_from_pdf
 from ae.core.skills.enter_text_using_selector import entertext
 from ae.core.skills.get_dom_with_content_type import get_dom_with_content_type
 from ae.core.skills.get_url import geturl
 from ae.core.skills.open_url import openurl
+from ae.core.skills.pdf_text_extractor import extract_text_from_pdf
+
 #from ae.core.skills.pdf_text_extractor import extract_text_from_pdf
 from ae.core.skills.press_key_combination import press_key_combination
 
 
-
 class BrowserNavAgent:
     def __init__(self, config_list, browser_nav_executor: autogen.UserProxyAgent): # type: ignore
         """
@@ -45,7 +43,9 @@ def __init__(self, config_list, browser_nav_executor: autogen.UserProxyAgent): #
             llm_config={
                 "config_list": config_list,
                 "cache_seed": None,
-                "temperature": 0.0
+                "temperature": 0.0,
+                "top_p": 0.001,
+                "seed":12345
             },
         )
         self.__register_skills()
@@ -70,9 +70,9 @@ def __register_skills(self):
         self.browser_nav_executor.register_for_execution()(openurl)
 
         # Register enter_text_and_click skill for LLM by assistant agent
-        self.agent.register_for_llm(description=LLM_PROMPTS["ENTER_TEXT_AND_CLICK_PROMPT"])(enter_text_and_click)
+        # self.agent.register_for_llm(description=LLM_PROMPTS["ENTER_TEXT_AND_CLICK_PROMPT"])(enter_text_and_click)
         # Register enter_text_and_click skill for execution by user_proxy_agent
-        self.browser_nav_executor.register_for_execution()(enter_text_and_click)
+        # self.browser_nav_executor.register_for_execution()(enter_text_and_click)
 
         # Register get_dom_with_content_type skill for LLM by assistant agent
         self.agent.register_for_llm(description=LLM_PROMPTS["GET_DOM_WITH_CONTENT_TYPE_PROMPT"])(get_dom_with_content_type)
@@ -120,5 +120,5 @@ def __register_skills(self):
             config={"callback": None},
         )
         '''
-        print(f">>> Function map: {self.browser_nav_executor.function_map}") # type: ignore
-        print(">>> Registered skills for BrowserNavAgent and BrowserNavExecutorAgent")
+        # print(f">>> Function map: {self.browser_nav_executor.function_map}") # type: ignore
+        # print(">>> Registered skills for BrowserNavAgent and BrowserNavExecutorAgent")

ae/core/agents/high_level_planner_agent.py

@@ -1,13 +1,15 @@
+from datetime import datetime
 from string import Template
-from ae.core.post_process_responses import final_reply_callback_planner_agent as print_message_as_planner  # type: ignore
+
 import autogen  # type: ignore
+from autogen import ConversableAgent  # type: ignore
+
 from ae.core.memory.static_ltm import get_user_ltm
-from ae.core.skills.get_user_input import get_user_input
+from ae.core.post_process_responses import final_reply_callback_planner_agent as print_message_as_planner  # type: ignore
 from ae.core.prompts import LLM_PROMPTS
-from datetime import datetime
-from autogen import Agent  # type: ignore
-from autogen import ConversableAgent # type: ignore
-from autogen import OpenAIWrapper  # type: ignore
+from ae.core.skills.get_user_input import get_user_input
+
+
 class PlannerAgent:
     def __init__(self, config_list, user_proxy_agent:ConversableAgent): # type: ignore
         """
@@ -21,7 +23,7 @@ def __init__(self, config_list, user_proxy_agent:ConversableAgent): # type: igno
 
         user_ltm = self.__get_ltm()
         system_message = LLM_PROMPTS["PLANNER_AGENT_PROMPT"]
-        print(f">>> Planner system_message: {system_message}")
+
         if user_ltm: #add the user LTM to the system prompt if it exists
             user_ltm = "\n" + user_ltm
             system_message = Template(system_message).substitute(basic_user_information=user_ltm)
@@ -32,7 +34,9 @@ def __init__(self, config_list, user_proxy_agent:ConversableAgent): # type: igno
             llm_config={
                 "config_list": config_list,
                 "cache_seed": None,
-                "temperature": 0.0
+                "temperature": 0.0,
+                "top_p": 0.001,
+                "seed":12345
             },
         )