From dcb18064a75365f1cfc19677a10637c41c12cc6a Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Wed, 18 Dec 2024 06:59:35 +0000 Subject: [PATCH 01/21] add list, create run kb --- naptha_sdk/cli.py | 86 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 483f6e7..5bca8eb 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -12,8 +12,7 @@ from naptha_sdk.client.hub import user_setup_flow from naptha_sdk.client.naptha import Naptha from naptha_sdk.schemas import AgentConfig, AgentDeployment, ChatCompletionRequest, EnvironmentDeployment, \ - OrchestratorDeployment, \ - OrchestratorRunInput, EnvironmentRunInput + OrchestratorDeployment, OrchestratorRunInput, EnvironmentRunInput, KBDeployment, KBRunInput from naptha_sdk.user import get_public_key load_dotenv(override=True) @@ -172,6 +171,34 @@ async def list_personas(naptha): print(tabulate(table_data, headers=headers, tablefmt="grid")) print(f"\nTotal personas: {len(personas)}") +async def list_knowledge_bases(naptha, knowledge_base_name=None): + knowledge_bases = await naptha.hub.list_knowledge_bases(knowledge_base_name=knowledge_base_name) + + if not knowledge_bases: + print("No knowledge bases found.") + return + + headers = ["Name", "ID", "Type", "Version", "Author", "Description", "URL"] + table_data = [] + + for knowledge_base in knowledge_bases: + # Wrap the description text for better readability + wrapped_description = '\n'.join(wrap(knowledge_base['description'], width=50)) + + row = [ + knowledge_base['name'], + knowledge_base['id'], + knowledge_base['type'], + knowledge_base['version'], + knowledge_base['author'], + wrapped_description, + knowledge_base['url'] + ] + table_data.append(row) + + print(tabulate(table_data, headers=headers, tablefmt="grid")) + print(f"\nTotal knowledge bases: {len(knowledge_bases)}") + async def create_agent(naptha, agent_config): print(f"Agent Config: {agent_config}") agent = await naptha.hub.create_agent(agent_config) @@ -219,6 +246,8 @@ async def create( module_type = "agent" elif "environment:" in module_name: module_type = "environment" + elif "knowledge_base:" in module_name: + module_type = "knowledge_base" else: module_type = "agent" @@ -286,6 +315,21 @@ async def create( result = await naptha.node.create(module_type, environment_deployment) print(f"Environment creation result: {result}") + elif module_type == "knowledge_base": + print("Creating Knowledge Base...") + if "knowledge_base:" in module_name: + module_name = module_name.split(":")[1] + else: + module_name = module_name + + knowledge_base_deployment = KBDeployment( + name=module_name, + module={"name": module_name}, + kb_node_url=os.getenv("NODE_URL") + ) + result = await naptha.node.create(module_type, knowledge_base_deployment) + print(f"Knowledge Base creation result: {result}") + async def run( naptha, module_name, @@ -308,6 +352,8 @@ async def run( module_type = "agent" elif "environment:" in module_name: module_type = "environment" + elif "knowledge_base:" in module_name: + module_type = "knowledge_base" else: module_type = "agent" # Default to agent for backwards compatibility @@ -379,7 +425,22 @@ async def run( consumer_id=user_id, ) environment_run = await naptha.node.run_environment_and_poll(environment_run_input) - + + elif module_type == "knowledge_base": + print("Running Knowledge Base...") + knowledge_base_deployment = KBDeployment( + name=module_name, + module={"name": module_name}, + kb_node_url=os.getenv("NODE_URL") + ) + + knowledge_base_run_input = KBRunInput( + consumer_id=user_id, + inputs=parameters, + kb_deployment=knowledge_base_deployment + ) + knowledge_base_run = await naptha.node.run_knowledge_base_and_poll(knowledge_base_run_input) + async def read_storage(naptha, hash_or_name, output_dir='./files', ipfs=False): """Read from storage, IPFS, or IPNS.""" try: @@ -450,6 +511,10 @@ async def main(): personas_parser.add_argument("-p", '--metadata', type=str, help='Metadata in "key=value" format') personas_parser.add_argument('-d', '--delete', action='store_true', help='Delete a persona') + # Knowledge base commands + knowledge_bases_parser = subparsers.add_parser("knowledge_bases", help="List available knowledge bases.") + knowledge_bases_parser.add_argument('knowledge_base_name', nargs='?', help='Optional knowledge base name') + # Create command create_parser = subparsers.add_parser("create", help="Execute create command.") create_parser.add_argument("module", help="Select the module to create") @@ -497,7 +562,7 @@ async def main(): args = _parse_str_args(args) if args.command == "signup": _, user_id = await user_setup_flow(hub_url, public_key) - elif args.command in ["nodes", "agents", "orchestrators", "environments", "personas", "run", "inference", "read_storage", "write_storage", "publish", "create"]: + elif args.command in ["nodes", "agents", "orchestrators", "environments", "personas", "knowledge_bases", "run", "inference", "read_storage", "write_storage", "publish", "create"]: if not naptha.hub.is_authenticated: if not hub_username or not hub_password: print( @@ -630,8 +695,21 @@ async def main(): await create_persona(naptha, persona_config) else: print("Invalid command.") + elif args.command == "knowledge_bases": + print(f"Knowledge base name: {args.knowledge_base_name}") + if not args.knowledge_base_name: + await list_knowledge_bases(naptha) + # get knowledge base by name + elif len(args.knowledge_base_name.split()) == 1: + await list_knowledge_bases(naptha, args.knowledge_base_name) + elif args.delete and len(args.knowledge_base_name.split()) == 1: + await naptha.hub.delete_knowledge_base(args.knowledge_base_name) + else: + print("Invalid command.") + elif args.command == "create": await create(naptha, args.module, args.agent_modules, args.worker_node_urls, args.environment_modules, args.environment_node_urls) + elif args.command == "run": if hasattr(args, 'parameters') and args.parameters is not None: try: From 8b7a0be6b5c52e1ebd1ce74da1f68b1ff20541f3 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Wed, 18 Dec 2024 06:59:53 +0000 Subject: [PATCH 02/21] list kb --- naptha_sdk/client/hub.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/naptha_sdk/client/hub.py b/naptha_sdk/client/hub.py index ff4485b..17686e5 100644 --- a/naptha_sdk/client/hub.py +++ b/naptha_sdk/client/hub.py @@ -150,6 +150,14 @@ async def list_personas(self, persona_name=None) -> List: persona = await self.surrealdb.query("SELECT * FROM persona WHERE id=$persona_name;", {"persona_name": persona_name}) return persona[0]['result'] + async def list_knowledge_bases(self, knowledge_base_name=None) -> List: + if not knowledge_base_name: + knowledge_bases = await self.surrealdb.query("SELECT * FROM knowledge_base;") + return knowledge_bases[0]['result'] + else: + knowledge_base = await self.surrealdb.query("SELECT * FROM knowledge_base WHERE name=$knowledge_base_name;", {"knowledge_base_name": knowledge_base_name}) + return knowledge_base[0]['result'] + async def delete_agent(self, agent_id: str) -> Tuple[bool, Optional[Dict]]: if ":" not in agent_id: agent_id = f"agent:{agent_id}".strip() From e8a384346e3e1e5a2e02e604b66010147a4157ed Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Wed, 18 Dec 2024 07:00:07 +0000 Subject: [PATCH 03/21] add kb related schemas --- naptha_sdk/schemas.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/naptha_sdk/schemas.py b/naptha_sdk/schemas.py index e360c02..01b62f6 100644 --- a/naptha_sdk/schemas.py +++ b/naptha_sdk/schemas.py @@ -60,6 +60,12 @@ class EnvironmentDeployment(BaseModel): environment_node_url: str environment_config: Optional[EnvironmentConfig] = EnvironmentConfig() +class KBDeployment(BaseModel): + name: Optional[str] = "kb_deployment" + module: Optional[Dict] = None + kb_node_url: Optional[str] = "http://localhost:7001" + kb_config: Optional[Dict] = None + class OrchestratorDeployment(BaseModel): name: Optional[str] = "orchestrator_deployment" module: Optional[Dict] = None @@ -181,6 +187,27 @@ class EnvironmentRun(BaseModel): duration: Optional[float] = None input_schema_ipfs_hash: Optional[str] = None +class KBRunInput(BaseModel): + consumer_id: str + inputs: Optional[Union[Dict, BaseModel, DockerParams]] = None + kb_deployment: KBDeployment + orchestrator_runs: List['OrchestratorRun'] = [] + +class KBRun(BaseModel): + consumer_id: str + inputs: Optional[Union[Dict, BaseModel, DockerParams]] = None + kb_deployment: KBDeployment + orchestrator_runs: List['OrchestratorRun'] = [] + status: str = "pending" + error: bool = False + id: Optional[str] = None + results: list[Optional[str]] = [] + error_message: Optional[str] = None + created_time: Optional[str] = None + start_processing_time: Optional[str] = None + completed_time: Optional[str] = None + duration: Optional[float] = None + class ChatMessage(BaseModel): role: str content: str From 20d0d4ef566d0405529bcb493c777919b5f68e22 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Wed, 18 Dec 2024 07:00:31 +0000 Subject: [PATCH 04/21] add list, create run kb --- naptha_sdk/client/node.py | 47 +++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/naptha_sdk/client/node.py b/naptha_sdk/client/node.py index 2a8de2c..4efccc2 100644 --- a/naptha_sdk/client/node.py +++ b/naptha_sdk/client/node.py @@ -19,7 +19,7 @@ from naptha_sdk.client import grpc_server_pb2 from naptha_sdk.client import grpc_server_pb2_grpc from naptha_sdk.schemas import AgentRun, AgentRunInput, ChatCompletionRequest, EnvironmentRun, EnvironmentRunInput, OrchestratorRun, \ - OrchestratorRunInput, AgentDeployment, EnvironmentDeployment, OrchestratorDeployment + OrchestratorRunInput, AgentDeployment, EnvironmentDeployment, OrchestratorDeployment, KBDeployment, KBRunInput, KBRun from naptha_sdk.utils import get_logger logger = get_logger(__name__) @@ -63,8 +63,6 @@ async def create(self, module_type: str, """ print(f"Creating {module_type}...") - print(f"Module Request: {module_request}") - print(f"Node URL: {self.node_url}") endpoint = f"{self.node_url}/{module_type}/create" try: @@ -93,14 +91,15 @@ async def create(self, module_type: str, print(f"An unexpected error occurred: {e}") raise - async def _run_and_poll(self, run_input: Union[AgentRunInput, EnvironmentRunInput, OrchestratorRunInput, Dict], module_type: str) -> Union[AgentRun, EnvironmentRun, OrchestratorRun, Dict]: + async def _run_and_poll(self, run_input: Union[AgentRunInput, EnvironmentRunInput, OrchestratorRunInput, KBRunInput, Dict], module_type: str) -> Union[AgentRun, EnvironmentRun, OrchestratorRun, KBRun, Dict]: """Generic method to run and poll either an agent, orchestrator, or environment. Args: - run_input: Either AgentRunInput, OrchestratorRunInput, or environment dict - module_type: Either 'agent', 'orchestrator', or 'environment' + run_input: Either AgentRunInput, OrchestratorRunInput, environment dict or KBDeployment + module_type: Either 'agent', 'orchestrator', 'environment' or 'knowledge_base' """ - + print(f"Run input: {run_input}") + print(f"Module type: {module_type}") # Start the run run = await getattr(self, f'run_{module_type}')(run_input) print(f"{module_type.title()} run started: {run}") @@ -108,8 +107,14 @@ async def _run_and_poll(self, run_input: Union[AgentRunInput, EnvironmentRunInpu current_results_len = 0 while True: # Check run status + if module_type == 'kb': + module_type = 'knowledge_base' + run = await getattr(self, f'check_{module_type}_run')(run) + if module_type == 'knowledge_base': + module_type = 'kb' + output = f"{run.status} {getattr(run, f'{module_type}_deployment').module['type']} {getattr(run, f'{module_type}_deployment').module['name']}" print(output) @@ -143,6 +148,10 @@ async def run_orchestrator_and_poll(self, orchestrator_run_input: OrchestratorRu async def run_environment_and_poll(self, environment_input: EnvironmentRunInput) -> EnvironmentRun: """Run an environment and poll for results until completion.""" return await self._run_and_poll(environment_input, 'environment') + + async def run_knowledge_base_and_poll(self, knowledge_base_input: KBDeployment) -> KBDeployment: + """Run a knowledge base and poll for results until completion.""" + return await self._run_and_poll(knowledge_base_input, 'knowledge_base') async def check_user_ws(self, user_input: Dict[str, str]): response = await self.send_receive_ws(user_input, "check_user") @@ -269,7 +278,8 @@ async def _run_module(self, run_input: Union[AgentRunInput, OrchestratorRunInput input_class = { 'agent': AgentRunInput, 'orchestrator': OrchestratorRunInput, - 'environment': EnvironmentRunInput + 'environment': EnvironmentRunInput, + 'knowledge_base': KBRunInput }[module_type] if isinstance(run_input, dict): @@ -292,7 +302,8 @@ async def _run_module(self, run_input: Union[AgentRunInput, OrchestratorRunInput return_class = { 'agent': AgentRun, 'orchestrator': OrchestratorRun, - 'environment': EnvironmentRun + 'environment': EnvironmentRun, + 'knowledge_base': KBRun }[module_type] return return_class(**json.loads(response.text)) except HTTPStatusError as e: @@ -427,16 +438,20 @@ async def run_environment(self, environment_run_input: EnvironmentRunInput) -> E """Run an environment on a node""" return await self._run_module(environment_run_input, 'environment') + async def run_knowledge_base(self, kb_run_input: KBRunInput) -> KBRun: + """Run a knowledge base on a node""" + return await self._run_module(kb_run_input, 'knowledge_base') + async def check_run( self, - module_run: Union[AgentRun, OrchestratorRun, EnvironmentRun], + module_run: Union[AgentRun, OrchestratorRun, EnvironmentRun, KBRun], module_type: str - ) -> Union[AgentRun, OrchestratorRun, EnvironmentRun]: + ) -> Union[AgentRun, OrchestratorRun, EnvironmentRun, KBRun]: """Generic method to check the status of a module run. Args: - module_run: Either AgentRun, OrchestratorRun, or EnvironmentRun object - module_type: Either 'agent', 'orchestrator', or 'environment' + module_run: Either AgentRun, OrchestratorRun, EnvironmentRun, or KBRun object + module_type: Either 'agent', 'orchestrator', 'environment', or 'knowledge_base' """ try: async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client: @@ -449,7 +464,8 @@ async def check_run( return_class = { 'agent': AgentRun, 'orchestrator': OrchestratorRun, - 'environment': EnvironmentRun + 'environment': EnvironmentRun, + 'knowledge_base': KBRun }[module_type] return return_class(**json.loads(response.text)) except HTTPStatusError as e: @@ -469,6 +485,9 @@ async def check_orchestrator_run(self, orchestrator_run: OrchestratorRun) -> Orc async def check_environment_run(self, environment_run: EnvironmentRun) -> EnvironmentRun: return await self.check_run(environment_run, 'environment') + async def check_knowledge_base_run(self, kb_run: KBRun) -> KBRun: + return await self.check_run(kb_run, 'knowledge_base') + async def create_agent_run(self, agent_run_input: AgentRunInput) -> AgentRun: try: async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client: From d2ed7d39dd95b370fc0a92fba6cc3d3866419d18 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Wed, 18 Dec 2024 08:38:00 +0000 Subject: [PATCH 05/21] AgentRunInput accepts empty kb_deployment --- naptha_sdk/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/naptha_sdk/schemas.py b/naptha_sdk/schemas.py index 01b62f6..a553e28 100644 --- a/naptha_sdk/schemas.py +++ b/naptha_sdk/schemas.py @@ -142,6 +142,7 @@ class AgentRunInput(BaseModel): inputs: Optional[Union[Dict, BaseModel, DockerParams]] = None agent_deployment: AgentDeployment environment_deployments: Optional[List[EnvironmentDeployment]] = None + kb_deployment: Optional[KBDeployment] = None orchestrator_runs: List['OrchestratorRun'] = [] class OrchestratorRunInput(BaseModel): From 46354377919ce7ea451e178bb747aec7706625d2 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Wed, 18 Dec 2024 08:38:21 +0000 Subject: [PATCH 06/21] KnowledgeBase abstraction --- naptha_sdk/knowledge_base.py | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 naptha_sdk/knowledge_base.py diff --git a/naptha_sdk/knowledge_base.py b/naptha_sdk/knowledge_base.py new file mode 100644 index 0000000..7b5681b --- /dev/null +++ b/naptha_sdk/knowledge_base.py @@ -0,0 +1,75 @@ +from naptha_sdk.client.node import Node +from naptha_sdk.schemas import KBDeployment, KBRunInput, KBRun +from typing import Any, Dict, List, Union +import logging + +logger = logging.getLogger(__name__) + +class KnowledgeBase: + def __init__(self, kb_deployment: KBDeployment): + self.kb_deployment = kb_deployment + self.kb_node = Node(self.kb_deployment.kb_node_url) + self.table_name = kb_deployment.kb_config.table_name + self.schema = kb_deployment.kb_config.schema + self.id_column = kb_deployment.kb_config.id_column + + if self.table_name is None: + self.table_name = kb_deployment.module["name"] + + @classmethod + async def create(cls, module_run): + instance = cls(module_run) + await instance._initialize() + return instance + + async def _initialize(self): + try: + tables = await self.kb_node.list_tables() + if self.table_name not in tables: + await self.kb_node.create_table(self.table_name, self.schema) + except Exception as e: + logger.error(f"Error initializing knowledge base: {str(e)}") + raise + + async def upsert_kb(self, run_id: str, data: Dict[str, Any]): + try: + # check if the run_id exists + existing_data = await self.kb_node.query_table( + self.table_name, + condition={self.id_column: run_id} + ) + + if existing_data["rows"]: + # update existing record + await self.kb_node.update_row( + self.table_name, + data=data, + condition={self.id_column: run_id} + ) + logger.info(f"Updated knowledge base with run_id: {run_id}") + else: + # insert new record + await self.kb_node.add_row( + self.table_name, + data={self.id_column: run_id, **data} + ) + logger.info(f"Inserted new knowledge base with run_id: {run_id}") + except Exception as e: + logger.error(f"Error upserting knowledge base: {str(e)}") + raise + + async def get_kb(self, run_id: str) -> Dict[str, Any]: + try: + data = await self.kb_node.query_table( + self.table_name, + condition={self.id_column: run_id} + ) + return data["rows"][0] if data["rows"] else None + except Exception as e: + logger.error(f"Error getting knowledge base: {str(e)}") + raise + + async def call_kb_func(self, kb_run_input: KBRunInput): + logger.info(f"Running knowledge base on knowledge base node {self.kb_node.node_url}") + kb_run = await self.kb_node.run_knowledge_base_and_poll(kb_run_input) + return kb_run From 363f1bef3decfec43b5e150db2f1075a57b4c6c4 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Wed, 18 Dec 2024 11:17:38 +0000 Subject: [PATCH 07/21] shorter knowledge bases command --- naptha_sdk/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 5bca8eb..0529c70 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -512,7 +512,7 @@ async def main(): personas_parser.add_argument('-d', '--delete', action='store_true', help='Delete a persona') # Knowledge base commands - knowledge_bases_parser = subparsers.add_parser("knowledge_bases", help="List available knowledge bases.") + knowledge_bases_parser = subparsers.add_parser("kbs", help="List available knowledge bases.") knowledge_bases_parser.add_argument('knowledge_base_name', nargs='?', help='Optional knowledge base name') # Create command @@ -562,7 +562,7 @@ async def main(): args = _parse_str_args(args) if args.command == "signup": _, user_id = await user_setup_flow(hub_url, public_key) - elif args.command in ["nodes", "agents", "orchestrators", "environments", "personas", "knowledge_bases", "run", "inference", "read_storage", "write_storage", "publish", "create"]: + elif args.command in ["nodes", "agents", "orchestrators", "environments", "personas", "kbs", "run", "inference", "read_storage", "write_storage", "publish", "create"]: if not naptha.hub.is_authenticated: if not hub_username or not hub_password: print( @@ -695,7 +695,7 @@ async def main(): await create_persona(naptha, persona_config) else: print("Invalid command.") - elif args.command == "knowledge_bases": + elif args.command == "kbs": print(f"Knowledge base name: {args.knowledge_base_name}") if not args.knowledge_base_name: await list_knowledge_bases(naptha) From 3289b7c8fcabb314653aafb30e4caf98649d1784 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Wed, 18 Dec 2024 12:54:31 +0000 Subject: [PATCH 08/21] update kb table name and metadata list order --- naptha_sdk/cli.py | 6 ++++-- naptha_sdk/client/hub.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 0529c70..d0e2b63 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -178,7 +178,7 @@ async def list_knowledge_bases(naptha, knowledge_base_name=None): print("No knowledge bases found.") return - headers = ["Name", "ID", "Type", "Version", "Author", "Description", "URL"] + headers = ["Name", "ID", "Author", "Description", "Module URL", "Module Type", "Module Version"] table_data = [] for knowledge_base in knowledge_bases: @@ -192,7 +192,9 @@ async def list_knowledge_bases(naptha, knowledge_base_name=None): knowledge_base['version'], knowledge_base['author'], wrapped_description, - knowledge_base['url'] + knowledge_base['module_url'], + knowledge_base['module_type'], + knowledge_base['module_version'], ] table_data.append(row) diff --git a/naptha_sdk/client/hub.py b/naptha_sdk/client/hub.py index 17686e5..5008ed8 100644 --- a/naptha_sdk/client/hub.py +++ b/naptha_sdk/client/hub.py @@ -152,10 +152,10 @@ async def list_personas(self, persona_name=None) -> List: async def list_knowledge_bases(self, knowledge_base_name=None) -> List: if not knowledge_base_name: - knowledge_bases = await self.surrealdb.query("SELECT * FROM knowledge_base;") + knowledge_bases = await self.surrealdb.query("SELECT * FROM kb;") return knowledge_bases[0]['result'] else: - knowledge_base = await self.surrealdb.query("SELECT * FROM knowledge_base WHERE name=$knowledge_base_name;", {"knowledge_base_name": knowledge_base_name}) + knowledge_base = await self.surrealdb.query("SELECT * FROM kb WHERE name=$knowledge_base_name;", {"knowledge_base_name": knowledge_base_name}) return knowledge_base[0]['result'] async def delete_agent(self, agent_id: str) -> Tuple[bool, Optional[Dict]]: From cfe0a235ce26e6dddf73a780a0ad40408eef29d8 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Wed, 18 Dec 2024 12:57:25 +0000 Subject: [PATCH 09/21] add list kb content --- naptha_sdk/cli.py | 11 ++++++++--- naptha_sdk/client/hub.py | 4 ++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index d0e2b63..fe92b0f 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -188,8 +188,6 @@ async def list_knowledge_bases(naptha, knowledge_base_name=None): row = [ knowledge_base['name'], knowledge_base['id'], - knowledge_base['type'], - knowledge_base['version'], knowledge_base['author'], wrapped_description, knowledge_base['module_url'], @@ -201,6 +199,10 @@ async def list_knowledge_bases(naptha, knowledge_base_name=None): print(tabulate(table_data, headers=headers, tablefmt="grid")) print(f"\nTotal knowledge bases: {len(knowledge_bases)}") +async def list_kb_content(naptha, knowledge_base_name): + content = await naptha.hub.list_kb_content(knowledge_base_name) + print(content) + async def create_agent(naptha, agent_config): print(f"Agent Config: {agent_config}") agent = await naptha.hub.create_agent(agent_config) @@ -516,6 +518,7 @@ async def main(): # Knowledge base commands knowledge_bases_parser = subparsers.add_parser("kbs", help="List available knowledge bases.") knowledge_bases_parser.add_argument('knowledge_base_name', nargs='?', help='Optional knowledge base name') + knowledge_bases_parser.add_argument('-l', '--list', action='store_true', help='List content in a knowledge base') # Create command create_parser = subparsers.add_parser("create", help="Execute create command.") @@ -702,8 +705,10 @@ async def main(): if not args.knowledge_base_name: await list_knowledge_bases(naptha) # get knowledge base by name - elif len(args.knowledge_base_name.split()) == 1: + elif len(args.knowledge_base_name.split()) == 1 and not args.list: await list_knowledge_bases(naptha, args.knowledge_base_name) + elif args.list: + await list_kb_content(naptha, args.knowledge_base_name) elif args.delete and len(args.knowledge_base_name.split()) == 1: await naptha.hub.delete_knowledge_base(args.knowledge_base_name) else: diff --git a/naptha_sdk/client/hub.py b/naptha_sdk/client/hub.py index 5008ed8..6900895 100644 --- a/naptha_sdk/client/hub.py +++ b/naptha_sdk/client/hub.py @@ -158,6 +158,10 @@ async def list_knowledge_bases(self, knowledge_base_name=None) -> List: knowledge_base = await self.surrealdb.query("SELECT * FROM kb WHERE name=$knowledge_base_name;", {"knowledge_base_name": knowledge_base_name}) return knowledge_base[0]['result'] + async def list_kb_content(self, knowledge_base_name: str) -> List: + kb_content = await self.surrealdb.query("SELECT * FROM kb_content WHERE kb_id=$kb_id;", {"kb_id": f"kb:{knowledge_base_name}"}) + return kb_content[0]['result'] + async def delete_agent(self, agent_id: str) -> Tuple[bool, Optional[Dict]]: if ":" not in agent_id: agent_id = f"agent:{agent_id}".strip() From 85b55c4b7c676c7b58e51a5f788c5e70144b2530 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Wed, 18 Dec 2024 22:59:25 +0000 Subject: [PATCH 10/21] list kb content from local db --- naptha_sdk/cli.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index fe92b0f..2904b9c 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -200,8 +200,14 @@ async def list_knowledge_bases(naptha, knowledge_base_name=None): print(f"\nTotal knowledge bases: {len(knowledge_bases)}") async def list_kb_content(naptha, knowledge_base_name): - content = await naptha.hub.list_kb_content(knowledge_base_name) - print(content) + rows = await naptha.node.query_table( + table_name=knowledge_base_name, + columns="*", + condition=None, + order_by=None, + limit=None + ) + print(rows) async def create_agent(naptha, agent_config): print(f"Agent Config: {agent_config}") From 963ffccf0e9634e09250b87e72b2b39e2c076fe7 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Wed, 18 Dec 2024 23:02:38 +0000 Subject: [PATCH 11/21] update readme --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/README.md b/README.md index 3181450..92f5dbd 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,52 @@ naptha environments -d environment_name naptha run environment:groupchat_environment -p "function_name='get_global_state'" ``` +## Knowledge Base Modules + +### Interact with the Knowledge Base Hub + +You can also use the CLI to explore available knowledge bases that you can use with agents: + +```bash +naptha kbs +``` + +### Register a New Knowledge Base Module on the Hub + +```bash +naptha kbs kb_name -p "description='Knowledge Base description' parameters='{input_parameter_1: str, input_parameter_2: int}' url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg' type='package' version='0.1' entrypoint='run.py'" +``` + +### Delete a Knowledge Base Module + +```bash +naptha kbs -d kb_name +``` + +### Create a New Knowledge Base on a Node + +```bash +naptha create kb:wikipedia_kb --kb_node_urls "http://node.naptha.ai:7001" +``` + +### List content in a Knowledge Base + +```bash +naptha kbs wikipedia_kb -l +``` + +### Add to a Knowledge Base + +```bash +naptha kbs wikipedia_kb -a "url='https://en.wikipedia.org/wiki/Elon_Musk' title='Elon Musk' text='Elon Musk is a billionaire entrepreneur and the CEO of SpaceX and Tesla.'" +``` + +### Run a Knowledge Base Module + +```bash +naptha run kb:wikipedia_kb -p "method_name='query_wikipedia' method_input_data='Elon Musk'" +``` + ## Personas ### Interact with the Persona Hub From 425117e83d575acf4ee7ebbaa78649fe6d38d785 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Thu, 19 Dec 2024 03:03:32 +0000 Subject: [PATCH 12/21] use rich when displaying listing modules --- naptha_sdk/cli.py | 312 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 220 insertions(+), 92 deletions(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 2904b9c..4bb6590 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -4,6 +4,9 @@ import os import shlex from textwrap import wrap +from rich.console import Console +from rich.table import Table +from rich import box import yaml from dotenv import load_dotenv @@ -35,169 +38,259 @@ async def list_nodes(naptha): nodes = await naptha.hub.list_nodes() if not nodes: - print("No nodes found.") + console = Console() + console.print("[red]No nodes found.[/red]") return - # Determine available keys - keys = list(nodes[0].keys()) + console = Console() + table = Table( + box=box.ROUNDED, + show_lines=True, + title="Available Nodes", + title_style="bold cyan", + header_style="bold blue", + row_styles=["", "dim"] # Alternating row styles + ) - # Create headers and table data based on available keys - headers = keys - table_data = [] + # Get dynamic headers from first node + headers = list(nodes[0].keys()) + + # Add columns with appropriate formatting + for header in headers: + table.add_column( + header, + overflow="fold", + max_width=60, + justify="left", + no_wrap=False + ) + # Add rows for node in nodes: - row = [] - for key in keys: - value = str(node.get(key, '')) - if len(value) > 50: - wrapped_value = '\n'.join(wrap(value, width=50)) - row.append(wrapped_value) - else: - row.append(value) - table_data.append(row) - - print("\nAll Nodes:") - print(tabulate(table_data, headers=headers, tablefmt="grid")) - print(f"\nTotal nodes: {len(nodes)}") + table.add_row(*[str(node.get(key, '')) for key in headers]) + + # Print table and summary + console.print() + console.print(table) + console.print(f"\n[green]Total nodes:[/green] {len(nodes)}") async def list_agents(naptha): agents = await naptha.hub.list_agents() if not agents: - print("No agents found.") + console = Console() + console.print("[red]No agents found.[/red]") return - headers = ["Name", "ID", "Type", "Version", "Author", "Parameters", "Description"] - table_data = [] + console = Console() + table = Table( + box=box.ROUNDED, + show_lines=True, + title="Available Agents", + title_style="bold cyan", + header_style="bold blue", + row_styles=["", "dim"] # Alternating row styles + ) + + # Define columns with specific formatting + table.add_column("Name", justify="left", style="green") + table.add_column("ID", justify="left") + table.add_column("Type", justify="left") + table.add_column("Version", justify="center") + table.add_column("Author", justify="left") + table.add_column("Parameters", justify="left", max_width=30) + table.add_column("Description", justify="left", max_width=50) + # Add rows for agent in agents: - # Wrap the description text - wrapped_description = '\n'.join(wrap(agent['description'], width=50)) - - row = [ + table.add_row( agent['name'], agent['id'], agent['type'], agent['version'], agent['author'], - agent['parameters'], - wrapped_description - ] - table_data.append(row) + str(agent['parameters']), + agent['description'] + ) - print(tabulate(table_data, headers=headers, tablefmt="grid")) - print(f"\nTotal agents: {len(agents)}") + # Print table and summary + console.print() + console.print(table) + console.print(f"\n[green]Total agents:[/green] {len(agents)}") async def list_orchestrators(naptha): orchestrators = await naptha.hub.list_orchestrators() if not orchestrators: - print("No orchestrators found.") + console = Console() + console.print("[red]No orchestrators found.[/red]") return - headers = ["Name", "ID", "Type", "Version", "Author", "Parameters", "Description"] - table_data = [] + console = Console() + table = Table( + box=box.ROUNDED, + show_lines=True, + title="Available Orchestrators", + title_style="bold cyan", + header_style="bold blue", + row_styles=["", "dim"] # Alternating row styles + ) + + # Define columns with specific formatting + table.add_column("Name", justify="left", style="green") + table.add_column("ID", justify="left") + table.add_column("Type", justify="left") + table.add_column("Version", justify="center") + table.add_column("Author", justify="left") + table.add_column("Parameters", justify="left", max_width=30) + table.add_column("Description", justify="left", max_width=50) + # Add rows for orchestrator in orchestrators: - # Wrap the description text - wrapped_description = '\n'.join(wrap(orchestrator['description'], width=50)) - - row = [ + table.add_row( orchestrator['name'], orchestrator['id'], orchestrator['type'], orchestrator['version'], orchestrator['author'], - orchestrator['parameters'], - wrapped_description - ] - table_data.append(row) + str(orchestrator['parameters']), + orchestrator['description'] + ) - print(tabulate(table_data, headers=headers, tablefmt="grid")) - print(f"\nTotal orchestrators: {len(orchestrators)}") + # Print table and summary + console.print() + console.print(table) + console.print(f"\n[green]Total orchestrators:[/green] {len(orchestrators)}") async def list_environments(naptha): environments = await naptha.hub.list_environments() if not environments: - print("No environments found.") + console = Console() + console.print("[red]No environments found.[/red]") return - headers = ["Name", "ID", "Type", "Version", "Author", "Parameters", "Description"] - table_data = [] + console = Console() + table = Table( + box=box.ROUNDED, + show_lines=True, + title="Available Environments", + title_style="bold cyan", + header_style="bold blue", + row_styles=["", "dim"] # Alternating row styles + ) + # Define columns with specific formatting + table.add_column("Name", justify="left", style="green") + table.add_column("ID", justify="left") + table.add_column("Type", justify="left") + table.add_column("Version", justify="center") + table.add_column("Author", justify="left") + table.add_column("Parameters", justify="left", max_width=30) + table.add_column("Description", justify="left", max_width=50) + + # Add rows for environment in environments: - # Wrap the description text - wrapped_description = '\n'.join(wrap(environment['description'], width=50)) - - row = [ + table.add_row( environment['name'], environment['id'], environment['type'], environment['version'], environment['author'], - environment['parameters'], - wrapped_description - ] - table_data.append(row) + str(environment['parameters']), + environment['description'] + ) - print(tabulate(table_data, headers=headers, tablefmt="grid")) - print(f"\nTotal environments: {len(environments)}") + # Print table and summary + console.print() + console.print(table) + console.print(f"\n[green]Total environments:[/green] {len(environments)}") async def list_personas(naptha): personas = await naptha.hub.list_personas() if not personas: - print("No personas found.") + console = Console() + console.print("[red]No personas found.[/red]") return - headers = ["Name", "ID", "Version", "Author", "Description", "URL"] - table_data = [] + console = Console() + table = Table( + box=box.ROUNDED, + show_lines=True, + title="Available Personas", + title_style="bold cyan", + header_style="bold blue", + row_styles=["", "dim"] # Alternating row styles + ) + + # Define columns with specific formatting + table.add_column("Name", justify="left", style="green") + table.add_column("ID", justify="left") + table.add_column("Version", justify="center") + table.add_column("Author", justify="left") + table.add_column("Description", justify="left", max_width=50) + table.add_column("URL", justify="left", max_width=40) + # Add rows for persona in personas: - # Wrap the description text - wrapped_description = '\n'.join(wrap(persona['description'], width=50)) - - row = [ + table.add_row( persona['name'], persona['id'], persona['version'], persona['author'], - wrapped_description, + persona['description'], persona['url'] - ] - table_data.append(row) + ) - print(tabulate(table_data, headers=headers, tablefmt="grid")) - print(f"\nTotal personas: {len(personas)}") + # Print table and summary + console.print() + console.print(table) + console.print(f"\n[green]Total personas:[/green] {len(personas)}") async def list_knowledge_bases(naptha, knowledge_base_name=None): knowledge_bases = await naptha.hub.list_knowledge_bases(knowledge_base_name=knowledge_base_name) if not knowledge_bases: - print("No knowledge bases found.") + console = Console() + console.print("[red]No knowledge bases found.[/red]") return - headers = ["Name", "ID", "Author", "Description", "Module URL", "Module Type", "Module Version"] - table_data = [] - - for knowledge_base in knowledge_bases: - # Wrap the description text for better readability - wrapped_description = '\n'.join(wrap(knowledge_base['description'], width=50)) - - row = [ - knowledge_base['name'], - knowledge_base['id'], - knowledge_base['author'], - wrapped_description, - knowledge_base['module_url'], - knowledge_base['module_type'], - knowledge_base['module_version'], - ] - table_data.append(row) - - print(tabulate(table_data, headers=headers, tablefmt="grid")) - print(f"\nTotal knowledge bases: {len(knowledge_bases)}") + console = Console() + table = Table( + box=box.ROUNDED, + show_lines=True, + title="Available Knowledge Bases", + title_style="bold cyan", + header_style="bold blue", + row_styles=["", "dim"] # Alternating row styles + ) + + # Define columns with specific formatting + table.add_column("Name", justify="left", style="green") + table.add_column("ID", justify="left") + table.add_column("Author", justify="left") + table.add_column("Description", justify="left", max_width=50) + table.add_column("Module URL", justify="left", max_width=40) + table.add_column("Module Type", justify="left") + table.add_column("Module Version", justify="center") + + # Add rows + for kb in knowledge_bases: + table.add_row( + kb['name'], + kb['id'], + kb['author'], + kb['description'], + kb['url'], + kb['type'], + kb['version'] + ) + + # Print table and summary + console.print() + console.print(table) + console.print(f"\n[green]Total knowledge bases:[/green] {len(knowledge_bases)}") async def list_kb_content(naptha, knowledge_base_name): rows = await naptha.node.query_table( @@ -207,7 +300,42 @@ async def list_kb_content(naptha, knowledge_base_name): order_by=None, limit=None ) - print(rows) + + if not rows.get('rows'): + console = Console() + console.print("[red]No content found in knowledge base.[/red]") + return + + console = Console() + table = Table( + box=box.ROUNDED, + show_lines=True, + title=f"Knowledge Base Content: {knowledge_base_name}", + title_style="bold cyan", + header_style="bold blue", + row_styles=["", "dim"] # Alternating row styles + ) + + # Add headers + headers = list(rows['rows'][0].keys()) + for header in headers: + if header.lower() in ['id', 'url']: + table.add_column(header, justify="left", max_width=40) + elif header.lower() in ['title', 'name']: + table.add_column(header, justify="left", style="green", max_width=40) + elif header.lower() in ['text', 'description', 'content']: + table.add_column(header, justify="left", max_width=60) + else: + table.add_column(header, justify="left", max_width=30) + + # Add rows + for row in rows['rows']: + table.add_row(*[str(row.get(key, '')) for key in headers]) + + # Print table and summary + console.print() + console.print(table) + console.print(f"\n[green]Total rows:[/green] {len(rows['rows'])}") async def create_agent(naptha, agent_config): print(f"Agent Config: {agent_config}") From edde7d72e73c858962143fb0c2597957acaa86d9 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Thu, 19 Dec 2024 07:17:55 +0000 Subject: [PATCH 13/21] use kb --- naptha_sdk/{knowledge_base.py => kb.py} | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) rename naptha_sdk/{knowledge_base.py => kb.py} (73%) diff --git a/naptha_sdk/knowledge_base.py b/naptha_sdk/kb.py similarity index 73% rename from naptha_sdk/knowledge_base.py rename to naptha_sdk/kb.py index 7b5681b..9332182 100644 --- a/naptha_sdk/knowledge_base.py +++ b/naptha_sdk/kb.py @@ -11,8 +11,10 @@ def __init__(self, kb_deployment: KBDeployment): self.kb_node = Node(self.kb_deployment.kb_node_url) self.table_name = kb_deployment.kb_config.table_name self.schema = kb_deployment.kb_config.schema - self.id_column = kb_deployment.kb_config.id_column - + if "id_column" in kb_deployment.kb_config: + self.id_column = kb_deployment.kb_config['id_column'] + else: + self.id_column = "id" if self.table_name is None: self.table_name = kb_deployment.module["name"] @@ -31,12 +33,12 @@ async def _initialize(self): logger.error(f"Error initializing knowledge base: {str(e)}") raise - async def upsert_kb(self, run_id: str, data: Dict[str, Any]): + async def upsert_kb(self, id_: str, data: Dict[str, Any]): try: - # check if the run_id exists + # check if the id_ exists existing_data = await self.kb_node.query_table( self.table_name, - condition={self.id_column: run_id} + condition={self.id_column: id_} ) if existing_data["rows"]: @@ -44,25 +46,25 @@ async def upsert_kb(self, run_id: str, data: Dict[str, Any]): await self.kb_node.update_row( self.table_name, data=data, - condition={self.id_column: run_id} + condition={self.id_column: id_} ) - logger.info(f"Updated knowledge base with run_id: {run_id}") + logger.info(f"Updated knowledge base with id: {id_}") else: # insert new record await self.kb_node.add_row( self.table_name, - data={self.id_column: run_id, **data} + data={self.id_column: id_, **data} ) - logger.info(f"Inserted new knowledge base with run_id: {run_id}") + logger.info(f"Inserted new knowledge base with id: {id_}") except Exception as e: logger.error(f"Error upserting knowledge base: {str(e)}") raise - async def get_kb(self, run_id: str) -> Dict[str, Any]: + async def get_kb(self, column_name: str, column_value: str) -> Dict[str, Any]: try: data = await self.kb_node.query_table( self.table_name, - condition={self.id_column: run_id} + condition={column_name: column_value} ) return data["rows"][0] if data["rows"] else None except Exception as e: @@ -71,5 +73,5 @@ async def get_kb(self, run_id: str) -> Dict[str, Any]: async def call_kb_func(self, kb_run_input: KBRunInput): logger.info(f"Running knowledge base on knowledge base node {self.kb_node.node_url}") - kb_run = await self.kb_node.run_knowledge_base_and_poll(kb_run_input) + kb_run = await self.kb_node.run_kb_and_poll(kb_run_input) return kb_run From aa00df3778ff519681849829fa704d91e627bebc Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Thu, 19 Dec 2024 07:18:24 +0000 Subject: [PATCH 14/21] use kb instad of knowledge_base --- naptha_sdk/cli.py | 116 +++++++++++++++++++++++++++----------- naptha_sdk/client/hub.py | 16 +++--- naptha_sdk/client/node.py | 29 ++++------ 3 files changed, 101 insertions(+), 60 deletions(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 4bb6590..52ffb0b 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -248,10 +248,10 @@ async def list_personas(naptha): console.print(table) console.print(f"\n[green]Total personas:[/green] {len(personas)}") -async def list_knowledge_bases(naptha, knowledge_base_name=None): - knowledge_bases = await naptha.hub.list_knowledge_bases(knowledge_base_name=knowledge_base_name) +async def list_kbs(naptha, kb_name=None): + kbs = await naptha.hub.list_kbs(kb_name=kb_name) - if not knowledge_bases: + if not kbs: console = Console() console.print("[red]No knowledge bases found.[/red]") return @@ -276,7 +276,7 @@ async def list_knowledge_bases(naptha, knowledge_base_name=None): table.add_column("Module Version", justify="center") # Add rows - for kb in knowledge_bases: + for kb in kbs: table.add_row( kb['name'], kb['id'], @@ -290,11 +290,11 @@ async def list_knowledge_bases(naptha, knowledge_base_name=None): # Print table and summary console.print() console.print(table) - console.print(f"\n[green]Total knowledge bases:[/green] {len(knowledge_bases)}") + console.print(f"\n[green]Total knowledge bases:[/green] {len(kbs)}") -async def list_kb_content(naptha, knowledge_base_name): +async def list_kb_content(naptha, kb_name): rows = await naptha.node.query_table( - table_name=knowledge_base_name, + table_name=kb_name, columns="*", condition=None, order_by=None, @@ -310,7 +310,7 @@ async def list_kb_content(naptha, knowledge_base_name): table = Table( box=box.ROUNDED, show_lines=True, - title=f"Knowledge Base Content: {knowledge_base_name}", + title=f"Knowledge Base Content: {kb_name}", title_style="bold cyan", header_style="bold blue", row_styles=["", "dim"] # Alternating row styles @@ -337,6 +337,47 @@ async def list_kb_content(naptha, knowledge_base_name): console.print(table) console.print(f"\n[green]Total rows:[/green] {len(rows['rows'])}") +async def add_data_to_kb(naptha, kb_name, data, user_id=None, kb_node_url="http://localhost:7001"): + try: + # Parse the data string into a dictionary + data_dict = {} + # Split by spaces, but keep quoted strings together + parts = shlex.split(data) + + for part in parts: + if '=' in part: + key, value = part.split('=', 1) + # Remove quotes if they exist + value = value.strip("'\"") + data_dict[key] = value + + data_dict = [data_dict] + + kb_run_input = { + "consumer_id": user_id, + "inputs": { + "mode": "add_data", + "data": json.dumps(data_dict) + }, + "kb_deployment": { + "name": kb_name, + "module": { + "name": kb_name + }, + "kb_node_url": kb_node_url + } + } + + kb_run = await naptha.node.run_kb_and_poll(kb_run_input) + console = Console() + console.print(f"\n[green]Successfully added data to knowledge base:[/green] {kb_name}") + console.print(kb_run) + + except Exception as e: + console = Console() + console.print(f"\n[red]Error adding data to knowledge base:[/red] {str(e)}") + + async def create_agent(naptha, agent_config): print(f"Agent Config: {agent_config}") agent = await naptha.hub.create_agent(agent_config) @@ -384,8 +425,8 @@ async def create( module_type = "agent" elif "environment:" in module_name: module_type = "environment" - elif "knowledge_base:" in module_name: - module_type = "knowledge_base" + elif "kb:" in module_name: + module_type = "kb" else: module_type = "agent" @@ -453,19 +494,19 @@ async def create( result = await naptha.node.create(module_type, environment_deployment) print(f"Environment creation result: {result}") - elif module_type == "knowledge_base": + elif module_type == "kb": print("Creating Knowledge Base...") - if "knowledge_base:" in module_name: + if "kb:" in module_name: module_name = module_name.split(":")[1] else: module_name = module_name - knowledge_base_deployment = KBDeployment( + kb_deployment = KBDeployment( name=module_name, module={"name": module_name}, kb_node_url=os.getenv("NODE_URL") ) - result = await naptha.node.create(module_type, knowledge_base_deployment) + result = await naptha.node.create(module_type, kb_deployment) print(f"Knowledge Base creation result: {result}") async def run( @@ -490,8 +531,8 @@ async def run( module_type = "agent" elif "environment:" in module_name: module_type = "environment" - elif "knowledge_base:" in module_name: - module_type = "knowledge_base" + elif "kb:" in module_name: + module_type = "kb" else: module_type = "agent" # Default to agent for backwards compatibility @@ -564,20 +605,20 @@ async def run( ) environment_run = await naptha.node.run_environment_and_poll(environment_run_input) - elif module_type == "knowledge_base": + elif module_type == "kb": print("Running Knowledge Base...") - knowledge_base_deployment = KBDeployment( + kb_deployment = KBDeployment( name=module_name, module={"name": module_name}, kb_node_url=os.getenv("NODE_URL") ) - knowledge_base_run_input = KBRunInput( + kb_run_input = KBRunInput( consumer_id=user_id, inputs=parameters, - kb_deployment=knowledge_base_deployment + kb_deployment=kb_deployment ) - knowledge_base_run = await naptha.node.run_knowledge_base_and_poll(knowledge_base_run_input) + kb_run = await naptha.node.run_kb_and_poll(kb_run_input) async def read_storage(naptha, hash_or_name, output_dir='./files', ipfs=False): """Read from storage, IPFS, or IPNS.""" @@ -650,9 +691,12 @@ async def main(): personas_parser.add_argument('-d', '--delete', action='store_true', help='Delete a persona') # Knowledge base commands - knowledge_bases_parser = subparsers.add_parser("kbs", help="List available knowledge bases.") - knowledge_bases_parser.add_argument('knowledge_base_name', nargs='?', help='Optional knowledge base name') - knowledge_bases_parser.add_argument('-l', '--list', action='store_true', help='List content in a knowledge base') + kbs_parser = subparsers.add_parser("kbs", help="List available knowledge bases.") + kbs_parser.add_argument('kb_name', nargs='?', help='Optional knowledge base name') + kbs_parser.add_argument('-l', '--list', action='store_true', help='List content in a knowledge base') + kbs_parser.add_argument('-a', '--add', action='store_true', help='Add data to a knowledge base') + kbs_parser.add_argument('-d', '--data', type=str, help='Data to add to a knowledge base', required=False) + kbs_parser.add_argument('-n', '--kb_node_url', type=str, help='Knowledge base node URL', default="http://localhost:7001") # Create command create_parser = subparsers.add_parser("create", help="Execute create command.") @@ -835,18 +879,22 @@ async def main(): else: print("Invalid command.") elif args.command == "kbs": - print(f"Knowledge base name: {args.knowledge_base_name}") - if not args.knowledge_base_name: - await list_knowledge_bases(naptha) - # get knowledge base by name - elif len(args.knowledge_base_name.split()) == 1 and not args.list: - await list_knowledge_bases(naptha, args.knowledge_base_name) + if not args.kb_name: + # List all knowledge bases + await list_kbs(naptha) elif args.list: - await list_kb_content(naptha, args.knowledge_base_name) - elif args.delete and len(args.knowledge_base_name.split()) == 1: - await naptha.hub.delete_knowledge_base(args.knowledge_base_name) + # List content of specific knowledge base + await list_kb_content(naptha, args.kb_name) + elif args.add: + # Add data to knowledge base + if not args.data: + console = Console() + console.print("[red]Data is required for add command.[/red]") + return + await add_data_to_kb(naptha, args.kb_name, args.data, user_id=user_id, kb_node_url=args.kb_node_url) else: - print("Invalid command.") + # Show specific knowledge base info + await list_kbs(naptha, args.kb_name) elif args.command == "create": await create(naptha, args.module, args.agent_modules, args.worker_node_urls, args.environment_modules, args.environment_node_urls) diff --git a/naptha_sdk/client/hub.py b/naptha_sdk/client/hub.py index 6900895..4d0f847 100644 --- a/naptha_sdk/client/hub.py +++ b/naptha_sdk/client/hub.py @@ -150,16 +150,16 @@ async def list_personas(self, persona_name=None) -> List: persona = await self.surrealdb.query("SELECT * FROM persona WHERE id=$persona_name;", {"persona_name": persona_name}) return persona[0]['result'] - async def list_knowledge_bases(self, knowledge_base_name=None) -> List: - if not knowledge_base_name: - knowledge_bases = await self.surrealdb.query("SELECT * FROM kb;") - return knowledge_bases[0]['result'] + async def list_kbs(self, kb_name=None) -> List: + if not kb_name: + kbs = await self.surrealdb.query("SELECT * FROM kb;") + return kbs[0]['result'] else: - knowledge_base = await self.surrealdb.query("SELECT * FROM kb WHERE name=$knowledge_base_name;", {"knowledge_base_name": knowledge_base_name}) - return knowledge_base[0]['result'] + kb = await self.surrealdb.query("SELECT * FROM kb WHERE name=$kb_name;", {"kb_name": kb_name}) + return kb[0]['result'] - async def list_kb_content(self, knowledge_base_name: str) -> List: - kb_content = await self.surrealdb.query("SELECT * FROM kb_content WHERE kb_id=$kb_id;", {"kb_id": f"kb:{knowledge_base_name}"}) + async def list_kb_content(self, kb_name: str) -> List: + kb_content = await self.surrealdb.query("SELECT * FROM kb_content WHERE kb_id=$kb_id;", {"kb_id": f"kb:{kb_name}"}) return kb_content[0]['result'] async def delete_agent(self, agent_id: str) -> Tuple[bool, Optional[Dict]]: diff --git a/naptha_sdk/client/node.py b/naptha_sdk/client/node.py index 4efccc2..05cb41d 100644 --- a/naptha_sdk/client/node.py +++ b/naptha_sdk/client/node.py @@ -96,7 +96,7 @@ async def _run_and_poll(self, run_input: Union[AgentRunInput, EnvironmentRunInpu Args: run_input: Either AgentRunInput, OrchestratorRunInput, environment dict or KBDeployment - module_type: Either 'agent', 'orchestrator', 'environment' or 'knowledge_base' + module_type: Either 'agent', 'orchestrator', 'environment' or 'kb' """ print(f"Run input: {run_input}") print(f"Module type: {module_type}") @@ -106,14 +106,7 @@ async def _run_and_poll(self, run_input: Union[AgentRunInput, EnvironmentRunInpu current_results_len = 0 while True: - # Check run status - if module_type == 'kb': - module_type = 'knowledge_base' - run = await getattr(self, f'check_{module_type}_run')(run) - - if module_type == 'knowledge_base': - module_type = 'kb' output = f"{run.status} {getattr(run, f'{module_type}_deployment').module['type']} {getattr(run, f'{module_type}_deployment').module['name']}" print(output) @@ -149,9 +142,9 @@ async def run_environment_and_poll(self, environment_input: EnvironmentRunInput) """Run an environment and poll for results until completion.""" return await self._run_and_poll(environment_input, 'environment') - async def run_knowledge_base_and_poll(self, knowledge_base_input: KBDeployment) -> KBDeployment: + async def run_kb_and_poll(self, kb_input: KBDeployment) -> KBDeployment: """Run a knowledge base and poll for results until completion.""" - return await self._run_and_poll(knowledge_base_input, 'knowledge_base') + return await self._run_and_poll(kb_input, 'kb') async def check_user_ws(self, user_input: Dict[str, str]): response = await self.send_receive_ws(user_input, "check_user") @@ -279,7 +272,7 @@ async def _run_module(self, run_input: Union[AgentRunInput, OrchestratorRunInput 'agent': AgentRunInput, 'orchestrator': OrchestratorRunInput, 'environment': EnvironmentRunInput, - 'knowledge_base': KBRunInput + 'kb': KBRunInput }[module_type] if isinstance(run_input, dict): @@ -303,7 +296,7 @@ async def _run_module(self, run_input: Union[AgentRunInput, OrchestratorRunInput 'agent': AgentRun, 'orchestrator': OrchestratorRun, 'environment': EnvironmentRun, - 'knowledge_base': KBRun + 'kb': KBRun }[module_type] return return_class(**json.loads(response.text)) except HTTPStatusError as e: @@ -438,9 +431,9 @@ async def run_environment(self, environment_run_input: EnvironmentRunInput) -> E """Run an environment on a node""" return await self._run_module(environment_run_input, 'environment') - async def run_knowledge_base(self, kb_run_input: KBRunInput) -> KBRun: + async def run_kb(self, kb_run_input: KBRunInput) -> KBRun: """Run a knowledge base on a node""" - return await self._run_module(kb_run_input, 'knowledge_base') + return await self._run_module(kb_run_input, 'kb') async def check_run( self, @@ -451,7 +444,7 @@ async def check_run( Args: module_run: Either AgentRun, OrchestratorRun, EnvironmentRun, or KBRun object - module_type: Either 'agent', 'orchestrator', 'environment', or 'knowledge_base' + module_type: Either 'agent', 'orchestrator', 'environment', or 'kb' """ try: async with httpx.AsyncClient(timeout=HTTP_TIMEOUT) as client: @@ -465,7 +458,7 @@ async def check_run( 'agent': AgentRun, 'orchestrator': OrchestratorRun, 'environment': EnvironmentRun, - 'knowledge_base': KBRun + 'kb': KBRun }[module_type] return return_class(**json.loads(response.text)) except HTTPStatusError as e: @@ -485,8 +478,8 @@ async def check_orchestrator_run(self, orchestrator_run: OrchestratorRun) -> Orc async def check_environment_run(self, environment_run: EnvironmentRun) -> EnvironmentRun: return await self.check_run(environment_run, 'environment') - async def check_knowledge_base_run(self, kb_run: KBRun) -> KBRun: - return await self.check_run(kb_run, 'knowledge_base') + async def check_kb_run(self, kb_run: KBRun) -> KBRun: + return await self.check_run(kb_run, 'kb') async def create_agent_run(self, agent_run_input: AgentRunInput) -> AgentRun: try: From 52cbe70d63f0aac49c052532a91a096a8336f371 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Thu, 19 Dec 2024 07:33:13 +0000 Subject: [PATCH 15/21] fix KnowledgeBase class --- naptha_sdk/kb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/naptha_sdk/kb.py b/naptha_sdk/kb.py index 9332182..f6512b4 100644 --- a/naptha_sdk/kb.py +++ b/naptha_sdk/kb.py @@ -9,8 +9,8 @@ class KnowledgeBase: def __init__(self, kb_deployment: KBDeployment): self.kb_deployment = kb_deployment self.kb_node = Node(self.kb_deployment.kb_node_url) - self.table_name = kb_deployment.kb_config.table_name - self.schema = kb_deployment.kb_config.schema + self.table_name = kb_deployment.kb_config['table_name'] + self.schema = kb_deployment.kb_config['schema'] if "id_column" in kb_deployment.kb_config: self.id_column = kb_deployment.kb_config['id_column'] else: From 393ff4d53c1991fcf209be237a1413b52b1f515d Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 19 Dec 2024 10:10:35 +0000 Subject: [PATCH 16/21] update kb module column names --- README.md | 2 +- naptha_sdk/cli.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 92f5dbd..2f12b38 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ naptha kbs ### Register a New Knowledge Base Module on the Hub ```bash -naptha kbs kb_name -p "description='Knowledge Base description' parameters='{input_parameter_1: str, input_parameter_2: int}' url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg' type='package' version='0.1' entrypoint='run.py'" +naptha kbs kb_name -p "description='Knowledge Base description' parameters='{input_parameter_1: str, input_parameter_2: int}' module_url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg'" ``` ### Delete a Knowledge Base Module diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 52ffb0b..5bd5939 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -282,9 +282,9 @@ async def list_kbs(naptha, kb_name=None): kb['id'], kb['author'], kb['description'], - kb['url'], - kb['type'], - kb['version'] + kb['module_url'], + kb['module_type'], + kb['module_version'] ) # Print table and summary From d0afb7c4d49052b2a3f70355721dde3fefebc1c0 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 19 Dec 2024 10:38:17 +0000 Subject: [PATCH 17/21] add parameters, create and delete kb --- naptha_sdk/cli.py | 33 ++++++++++++++++++++++++++++++++- naptha_sdk/client/hub.py | 17 +++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 5bd5939..c20741b 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -271,6 +271,7 @@ async def list_kbs(naptha, kb_name=None): table.add_column("ID", justify="left") table.add_column("Author", justify="left") table.add_column("Description", justify="left", max_width=50) + table.add_column("Parameters", justify="left", max_width=40) table.add_column("Module URL", justify="left", max_width=40) table.add_column("Module Type", justify="left") table.add_column("Module Version", justify="center") @@ -282,6 +283,7 @@ async def list_kbs(naptha, kb_name=None): kb['id'], kb['author'], kb['description'], + kb['parameters'], kb['module_url'], kb['module_type'], kb['module_version'] @@ -693,9 +695,11 @@ async def main(): # Knowledge base commands kbs_parser = subparsers.add_parser("kbs", help="List available knowledge bases.") kbs_parser.add_argument('kb_name', nargs='?', help='Optional knowledge base name') + kbs_parser.add_argument('-p', '--metadata', type=str, help='Metadata for knowledge base registration in "key=value" format') + kbs_parser.add_argument('-d', '--delete', action='store_true', help='Delete a knowledge base') kbs_parser.add_argument('-l', '--list', action='store_true', help='List content in a knowledge base') kbs_parser.add_argument('-a', '--add', action='store_true', help='Add data to a knowledge base') - kbs_parser.add_argument('-d', '--data', type=str, help='Data to add to a knowledge base', required=False) + kbs_parser.add_argument('-t', '--data', type=str, help='Data to add to a knowledge base', required=False) kbs_parser.add_argument('-n', '--kb_node_url', type=str, help='Knowledge base node URL', default="http://localhost:7001") # Create command @@ -892,6 +896,33 @@ async def main(): console.print("[red]Data is required for add command.[/red]") return await add_data_to_kb(naptha, args.kb_name, args.data, user_id=user_id, kb_node_url=args.kb_node_url) + elif args.delete and len(args.kb_name.split()) == 1: + await naptha.hub.delete_kb(args.kb_name) + elif len(args.kb_name.split()) == 1: + if hasattr(args, 'metadata') and args.metadata is not None: + params = shlex.split(args.metadata) + parsed_params = {} + for param in params: + key, value = param.split('=') + parsed_params[key] = value + + required_metadata = ['description', 'parameters', 'module_url'] + if not all(param in parsed_params for param in required_metadata): + print(f"Missing one or more of the following required metadata: {required_metadata}") + return + + kb_config = { + "id": f"kb:{args.kb_name}", + "name": args.kb_name, + "description": parsed_params['description'], + "parameters": parsed_params['parameters'], + "author": naptha.hub.user_id, + "module_url": parsed_params['module_url'], + "module_type": parsed_params.get('module_type', 'package'), + "module_version": parsed_params.get('module_version', '0.1'), + "module_entrypoint": parsed_params.get('module_entrypoint', 'run.py') + } + await naptha.hub.create_kb(kb_config) else: # Show specific knowledge base info await list_kbs(naptha, args.kb_name) diff --git a/naptha_sdk/client/hub.py b/naptha_sdk/client/hub.py index 4d0f847..ba23a57 100644 --- a/naptha_sdk/client/hub.py +++ b/naptha_sdk/client/hub.py @@ -206,6 +206,17 @@ async def delete_persona(self, persona_id: str) -> Tuple[bool, Optional[Dict]]: print("Failed to delete persona") return success + async def delete_kb(self, kb_id: str) -> Tuple[bool, Optional[Dict]]: + if ":" not in kb_id: + kb_id = f"kb:{kb_id}".strip() + print(f"Deleting knowledge base: {kb_id}") + success = await self.surrealdb.delete(kb_id) + if success: + print("Deleted knowledge base") + else: + print("Failed to delete knowledge base") + return success + async def create_agent(self, agent_config: Dict) -> Tuple[bool, Optional[Dict]]: if not agent_config.get('id'): return await self.surrealdb.create("agent", agent_config) @@ -230,6 +241,12 @@ async def create_persona(self, persona_config: Dict) -> Tuple[bool, Optional[Dic else: return await self.surrealdb.create(persona_config.pop('id'), persona_config) + async def create_kb(self, kb_config: Dict) -> Tuple[bool, Optional[Dict]]: + if not kb_config.get('id'): + return await self.surrealdb.create("kb", kb_config) + else: + return await self.surrealdb.create(kb_config.pop('id'), kb_config) + async def update_agent(self, agent_config: Dict) -> Tuple[bool, Optional[Dict]]: return await self.surrealdb.update("agent", agent_config) From 6e1945f53f51f31e3b8216b6ec4a9bcc2e864d53 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 19 Dec 2024 14:39:41 +0000 Subject: [PATCH 18/21] update module field names --- README.md | 12 ++--- naptha_sdk/cli.py | 87 ++++++++++++++++++++----------------- naptha_sdk/client/naptha.py | 12 ++--- naptha_sdk/client/node.py | 4 +- naptha_sdk/configs.py | 2 +- 5 files changed, 63 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 2f12b38..0e22b83 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ For each agent, you will see a url where you can check out the code. ### Create a New Agent ```bash -naptha agents agent_name -p "description='Agent description' parameters='{tool_name: str, tool_input_data: str}' url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg' type='package' version='0.1'" +naptha agents agent_name -p "description='Agent description' parameters='{tool_name: str, tool_input_data: str}' module_url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg'" ``` ### Delete an Agent @@ -136,7 +136,7 @@ For each orchestrator, you will see a url where you can check out the code. ### Create a New Agent Orchestrator ```bash -naptha orchestrators orchestrator_name -p "description='Orchestrator description' parameters='{input_parameter_1: str, input_parameter_2: int}' url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg' type='package' version='0.1'" +naptha orchestrators orchestrator_name -p "description='Orchestrator description' parameters='{input_parameter_1: str, input_parameter_2: int}' module_url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg'" ``` ### Delete an Agent Orchestrator @@ -201,7 +201,7 @@ naptha environments ### Create a New Environment Module ```bash -naptha environments environment_name -p "description='Environment description' parameters='{input_parameter_1: str, input_parameter_2: int}' url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg' type='package' version='0.1' entrypoint='run.py'" +naptha environments environment_name -p "description='Environment description' parameters='{input_parameter_1: str, input_parameter_2: int}' module_url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg'" ``` ### Delete an Environment Module @@ -241,7 +241,7 @@ naptha kbs -d kb_name ### Create a New Knowledge Base on a Node ```bash -naptha create kb:wikipedia_kb --kb_node_urls "http://node.naptha.ai:7001" +naptha create kb:wikipedia_kb ``` ### List content in a Knowledge Base @@ -253,7 +253,7 @@ naptha kbs wikipedia_kb -l ### Add to a Knowledge Base ```bash -naptha kbs wikipedia_kb -a "url='https://en.wikipedia.org/wiki/Elon_Musk' title='Elon Musk' text='Elon Musk is a billionaire entrepreneur and the CEO of SpaceX and Tesla.'" +naptha kbs wikipedia_kb -a "module_url='https://en.wikipedia.org/wiki/Elon_Musk' title='Elon Musk' text='Elon Musk is a billionaire entrepreneur and the CEO of SpaceX and Tesla.'" ``` ### Run a Knowledge Base Module @@ -277,7 +277,7 @@ For each persona, you will see a url where you can check out the data. ### Create a New Persona ```bash -naptha personas persona_name -p "description='Persona description' url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg' version='0.1'" +naptha personas persona_name -p "description='Persona description' module_url='ipfs://QmNer9SRKmJPv4Ae3vdVYo6eFjPcyJ8uZ2rRSYd3koT6jg'" ``` ### Delete a Persona diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index c20741b..869b165 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -95,22 +95,24 @@ async def list_agents(naptha): # Define columns with specific formatting table.add_column("Name", justify="left", style="green") table.add_column("ID", justify="left") - table.add_column("Type", justify="left") - table.add_column("Version", justify="center") table.add_column("Author", justify="left") - table.add_column("Parameters", justify="left", max_width=30) table.add_column("Description", justify="left", max_width=50) + table.add_column("Parameters", justify="left", max_width=30) + table.add_column("Module URL", justify="left", max_width=30) + table.add_column("Module Type", justify="left") + table.add_column("Module Version", justify="center") # Add rows for agent in agents: table.add_row( agent['name'], agent['id'], - agent['type'], - agent['version'], agent['author'], + agent['description'], str(agent['parameters']), - agent['description'] + agent['module_url'], + agent['module_type'], + agent['module_version'], ) # Print table and summary @@ -139,22 +141,24 @@ async def list_orchestrators(naptha): # Define columns with specific formatting table.add_column("Name", justify="left", style="green") table.add_column("ID", justify="left") - table.add_column("Type", justify="left") - table.add_column("Version", justify="center") table.add_column("Author", justify="left") - table.add_column("Parameters", justify="left", max_width=30) table.add_column("Description", justify="left", max_width=50) + table.add_column("Parameters", justify="left", max_width=30) + table.add_column("Module URL", justify="left", max_width=30) + table.add_column("Module Type", justify="left") + table.add_column("Module Version", justify="center") # Add rows for orchestrator in orchestrators: table.add_row( orchestrator['name'], orchestrator['id'], - orchestrator['type'], - orchestrator['version'], orchestrator['author'], + orchestrator['description'], str(orchestrator['parameters']), - orchestrator['description'] + orchestrator['module_url'], + orchestrator['module_type'], + orchestrator['module_version'], ) # Print table and summary @@ -183,22 +187,24 @@ async def list_environments(naptha): # Define columns with specific formatting table.add_column("Name", justify="left", style="green") table.add_column("ID", justify="left") - table.add_column("Type", justify="left") - table.add_column("Version", justify="center") table.add_column("Author", justify="left") - table.add_column("Parameters", justify="left", max_width=30) table.add_column("Description", justify="left", max_width=50) + table.add_column("Parameters", justify="left", max_width=30) + table.add_column("Module URL", justify="left", max_width=30) + table.add_column("Module Type", justify="left") + table.add_column("Module Version", justify="center") # Add rows for environment in environments: table.add_row( environment['name'], environment['id'], - environment['type'], - environment['version'], environment['author'], + environment['description'], str(environment['parameters']), - environment['description'] + environment['module_url'], + environment['module_type'], + environment['module_version'], ) # Print table and summary @@ -227,20 +233,20 @@ async def list_personas(naptha): # Define columns with specific formatting table.add_column("Name", justify="left", style="green") table.add_column("ID", justify="left") - table.add_column("Version", justify="center") table.add_column("Author", justify="left") table.add_column("Description", justify="left", max_width=50) - table.add_column("URL", justify="left", max_width=40) + table.add_column("Module URL", justify="left", max_width=40) + table.add_column("Module Version", justify="center") # Add rows for persona in personas: table.add_row( persona['name'], persona['id'], - persona['version'], persona['author'], persona['description'], - persona['url'] + persona['module_url'], + persona['module_version'], ) # Print table and summary @@ -321,7 +327,7 @@ async def list_kb_content(naptha, kb_name): # Add headers headers = list(rows['rows'][0].keys()) for header in headers: - if header.lower() in ['id', 'url']: + if header.lower() in ['id', 'module_url']: table.add_column(header, justify="left", max_width=40) elif header.lower() in ['title', 'name']: table.add_column(header, justify="left", style="green", max_width=40) @@ -553,7 +559,7 @@ async def run( name=module_name, module={"name": module_name}, worker_node_url=worker_node_urls[0], - agent_config=AgentConfig(persona_module={"url": personas_urls}) + agent_config=AgentConfig(persona_module={"module_url": personas_urls}) ) agent_run_input = { @@ -772,7 +778,7 @@ async def main(): key, value = param.split('=') parsed_params[key] = value - required_metadata = ['description', 'parameters', 'url', 'type', 'version'] + required_metadata = ['description', 'parameters', 'module_url'] missing_metadata = [param for param in required_metadata if param not in parsed_params] if missing_metadata: print(f"Missing required metadata: {', '.join(missing_metadata)}") @@ -784,9 +790,10 @@ async def main(): "description": parsed_params['description'], "parameters": parsed_params['parameters'], "author": naptha.hub.user_id, - "url": parsed_params['url'], - "type": parsed_params['type'], - "version": parsed_params['version'], + "module_url": parsed_params['module_url'], + "module_type": parsed_params.get('module_type', 'package'), + "module_version": parsed_params.get('module_version', '0.1'), + "module_entrypoint": parsed_params.get('module_entrypoint', 'run.py') } await create_agent(naptha, agent_config) else: @@ -804,7 +811,7 @@ async def main(): key, value = param.split('=') parsed_params[key] = value - required_metadata = ['description', 'parameters', 'url', 'type', 'version'] + required_metadata = ['description', 'parameters', 'module_url'] if not all(param in parsed_params for param in required_metadata): print(f"Missing one or more of the following required metadata: {required_metadata}") return @@ -815,9 +822,10 @@ async def main(): "description": parsed_params['description'], "parameters": parsed_params['parameters'], "author": naptha.hub.user_id, - "url": parsed_params['url'], - "type": parsed_params['type'], - "version": parsed_params['version'], + "module_url": parsed_params['module_url'], + "module_type": parsed_params.get('module_type', 'package'), + "module_version": parsed_params.get('module_version', '0.1'), + "module_entrypoint": parsed_params.get('module_entrypoint', 'run.py') } await create_orchestrator(naptha, orchestrator_config) else: @@ -835,7 +843,7 @@ async def main(): key, value = param.split('=') parsed_params[key] = value - required_metadata = ['description', 'parameters', 'url', 'type', 'version'] + required_metadata = ['description', 'parameters', 'module_url'] if not all(param in parsed_params for param in required_metadata): print(f"Missing one or more of the following required metadata: {required_metadata}") return @@ -846,9 +854,10 @@ async def main(): "description": parsed_params['description'], "parameters": parsed_params['parameters'], "author": naptha.hub.user_id, - "url": parsed_params['url'], - "type": parsed_params['type'], - "version": parsed_params['version'], + "module_url": parsed_params['module_url'], + "module_type": parsed_params.get('module_type', 'package'), + "module_version": parsed_params.get('module_version', '0.1'), + "module_entrypoint": parsed_params.get('module_entrypoint', 'run.py') } await create_environment(naptha, environment_config) else: @@ -866,7 +875,7 @@ async def main(): key, value = param.split('=') parsed_params[key] = value - required_metadata = ['description', 'parameters', 'url', 'type', 'version'] + required_metadata = ['description', 'module_url'] if not all(param in parsed_params for param in required_metadata): print(f"Missing one or more of the following required metadata: {required_metadata}") return @@ -876,8 +885,8 @@ async def main(): "name": args.persona_name, "description": parsed_params['description'], "author": naptha.hub.user_id, - "url": parsed_params['url'], - "version": parsed_params['version'], + "module_url": parsed_params['module_url'], + "module_version": parsed_params.get('module_version', '0.1'), } await create_persona(naptha, persona_config) else: diff --git a/naptha_sdk/client/naptha.py b/naptha_sdk/client/naptha.py index 66b773f..b21cff3 100644 --- a/naptha_sdk/client/naptha.py +++ b/naptha_sdk/client/naptha.py @@ -56,9 +56,9 @@ async def create_agent(self, name): "name": name, "description": name, "author": self.hub.user_id, - "url": "None", - "type": "package", - "version": "0.1" + "module_url": "None", + "module_type": "package", + "module_version": "0.1" } logger.info(f"Registering Agent {agent_config}") agent = await self.hub.create_or_update_agent(agent_config) @@ -89,9 +89,9 @@ async def publish_agents(self): "description": agent, "parameters": agent, "author": self.hub.user_id, - "url": f'ipfs://{response["ipfs_hash"]}', - "type": "package", - "version": "0.1" + "module_url": f'ipfs://{response["ipfs_hash"]}', + "module_type": "package", + "module_version": "0.1" } logger.info(f"Registering Agent {agent_config}") agent = await self.hub.create_or_update_agent(agent_config) diff --git a/naptha_sdk/client/node.py b/naptha_sdk/client/node.py index 05cb41d..9fb51b4 100644 --- a/naptha_sdk/client/node.py +++ b/naptha_sdk/client/node.py @@ -54,7 +54,7 @@ def __init__(self, node_url: Optional[str] = None, indirect_node_id: Optional[st logger.info(f"Node URL: {node_url}") async def create(self, module_type: str, - module_request: Union[AgentDeployment, EnvironmentDeployment, OrchestratorDeployment]): + module_request: Union[AgentDeployment, EnvironmentDeployment, KBDeployment, OrchestratorDeployment]): """Generic method to create either an agent, orchestrator, or environment. Args: @@ -108,7 +108,7 @@ async def _run_and_poll(self, run_input: Union[AgentRunInput, EnvironmentRunInpu while True: run = await getattr(self, f'check_{module_type}_run')(run) - output = f"{run.status} {getattr(run, f'{module_type}_deployment').module['type']} {getattr(run, f'{module_type}_deployment').module['name']}" + output = f"{run.status} {getattr(run, f'{module_type}_deployment').module['module_type']} {getattr(run, f'{module_type}_deployment').module['name']}" print(output) results = run.results diff --git a/naptha_sdk/configs.py b/naptha_sdk/configs.py index f235db8..fa96642 100644 --- a/naptha_sdk/configs.py +++ b/naptha_sdk/configs.py @@ -21,7 +21,7 @@ def load_agent_deployments(agent_deployments_path, load_persona_data=True, load_ deployment["agent_config"]["llm_config"] = llm_config if load_persona_data: - persona_data, input_schema = load_persona(deployment["agent_config"]["persona_module"]["url"]) + persona_data, input_schema = load_persona(deployment["agent_config"]["persona_module"]["module_url"]) deployment["agent_config"]["persona_module"]["data"] = persona_data if load_persona_schema: deployment["agent_config"]["persona_module"]["data"] = input_schema(**persona_data) From 4fcb6020af31a9fafbdff45ae5b33084159e5493 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 19 Dec 2024 19:21:26 +0000 Subject: [PATCH 19/21] instructions for kb working --- README.md | 16 +++++++++++----- naptha_sdk/cli.py | 6 +++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0e22b83..07aabc0 100644 --- a/README.md +++ b/README.md @@ -244,22 +244,28 @@ naptha kbs -d kb_name naptha create kb:wikipedia_kb ``` -### List content in a Knowledge Base +### Initialize the content in the Knowledge Base + +```bash +naptha run kb:wikipedia_kb -p "mode='init'" +``` + +### List content in the Knowledge Base ```bash naptha kbs wikipedia_kb -l ``` -### Add to a Knowledge Base +### Add to the Knowledge Base ```bash -naptha kbs wikipedia_kb -a "module_url='https://en.wikipedia.org/wiki/Elon_Musk' title='Elon Musk' text='Elon Musk is a billionaire entrepreneur and the CEO of SpaceX and Tesla.'" +naptha kbs wikipedia_kb -a -c "url='https://en.wikipedia.org/wiki/Socrates' title='Socrates' text='Socrates was a Greek philosopher from Athens who is credited as the founder of Western philosophy and as among the first moral philosophers of the ethical tradition of thought.'" ``` -### Run a Knowledge Base Module +### Query the Knowledge Base Module ```bash -naptha run kb:wikipedia_kb -p "method_name='query_wikipedia' method_input_data='Elon Musk'" +naptha run kb:wikipedia_kb -p "mode='query' query='Socrates'" ``` ## Personas diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 869b165..1898bff 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -705,7 +705,7 @@ async def main(): kbs_parser.add_argument('-d', '--delete', action='store_true', help='Delete a knowledge base') kbs_parser.add_argument('-l', '--list', action='store_true', help='List content in a knowledge base') kbs_parser.add_argument('-a', '--add', action='store_true', help='Add data to a knowledge base') - kbs_parser.add_argument('-t', '--data', type=str, help='Data to add to a knowledge base', required=False) + kbs_parser.add_argument('-c', '--content', type=str, help='Content to add to a knowledge base', required=False) kbs_parser.add_argument('-n', '--kb_node_url', type=str, help='Knowledge base node URL', default="http://localhost:7001") # Create command @@ -900,11 +900,11 @@ async def main(): await list_kb_content(naptha, args.kb_name) elif args.add: # Add data to knowledge base - if not args.data: + if not args.content: console = Console() console.print("[red]Data is required for add command.[/red]") return - await add_data_to_kb(naptha, args.kb_name, args.data, user_id=user_id, kb_node_url=args.kb_node_url) + await add_data_to_kb(naptha, args.kb_name, args.content, user_id=user_id, kb_node_url=args.kb_node_url) elif args.delete and len(args.kb_name.split()) == 1: await naptha.hub.delete_kb(args.kb_name) elif len(args.kb_name.split()) == 1: From 61fb63781d3f1860f45ab49b9b26e7d25e7b7ce2 Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 19 Dec 2024 19:57:15 +0000 Subject: [PATCH 20/21] add kb deployment to agent deployments --- README.md | 6 ++++++ naptha_sdk/cli.py | 16 ++++++++++++---- naptha_sdk/schemas.py | 13 +++++++------ 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 07aabc0..7c7cf1a 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,12 @@ naptha kbs wikipedia_kb -a -c "url='https://en.wikipedia.org/wiki/Socrates' titl naptha run kb:wikipedia_kb -p "mode='query' query='Socrates'" ``` +### Run an Agent that interacts with the Knowledge Base + +```bash +naptha run agent:wikipedia_agent -p "query='socrates' question='Who is Socrates?'" --kb_node_urls "http://localhost:7001" +``` + ## Personas ### Interact with the Persona Hub diff --git a/naptha_sdk/cli.py b/naptha_sdk/cli.py index 1898bff..42d7416 100644 --- a/naptha_sdk/cli.py +++ b/naptha_sdk/cli.py @@ -524,6 +524,7 @@ async def run( parameters=None, worker_node_urls="http://localhost:7001", environment_node_urls=["http://localhost:7001"], + kb_node_urls=["http://localhost:7001"], yaml_file=None, personas_urls=None ): @@ -555,11 +556,17 @@ async def run( if module_type == "agent": print("Running Agent...") + + kb_deployments = [] + for kb_node_url in kb_node_urls: + kb_deployments.append(KBDeployment(kb_node_url=kb_node_url)) + agent_deployment = AgentDeployment( name=module_name, module={"name": module_name}, worker_node_url=worker_node_urls[0], - agent_config=AgentConfig(persona_module={"module_url": personas_urls}) + agent_config=AgentConfig(persona_module={"module_url": personas_urls}), + kb_deployments=kb_deployments ) agent_run_input = { @@ -706,7 +713,7 @@ async def main(): kbs_parser.add_argument('-l', '--list', action='store_true', help='List content in a knowledge base') kbs_parser.add_argument('-a', '--add', action='store_true', help='Add data to a knowledge base') kbs_parser.add_argument('-c', '--content', type=str, help='Content to add to a knowledge base', required=False) - kbs_parser.add_argument('-n', '--kb_node_url', type=str, help='Knowledge base node URL', default="http://localhost:7001") + kbs_parser.add_argument('-k', '--kb_node_urls', type=str, help='Knowledge base node URLs', default=["http://localhost:7001"]) # Create command create_parser = subparsers.add_parser("create", help="Execute create command.") @@ -722,6 +729,7 @@ async def main(): run_parser.add_argument("-p", '--parameters', type=str, help='Parameters in "key=value" format') run_parser.add_argument("-n", "--worker_node_urls", help="Worker nodes to take part in agent runs.") run_parser.add_argument("-e", "--environment_node_urls", help="Environment nodes to store data during agent runs.") + run_parser.add_argument('-k', '--kb_node_urls', type=str, help='Knowledge base node URLs', default=["http://localhost:7001"]) run_parser.add_argument("-u", "--personas_urls", help="Personas URLs to install before running the agent") run_parser.add_argument("-f", "--file", help="YAML file with agent run parameters") @@ -904,7 +912,7 @@ async def main(): console = Console() console.print("[red]Data is required for add command.[/red]") return - await add_data_to_kb(naptha, args.kb_name, args.content, user_id=user_id, kb_node_url=args.kb_node_url) + await add_data_to_kb(naptha, args.kb_name, args.content, user_id=user_id, kb_node_url=args.kb_node_urls[0]) elif args.delete and len(args.kb_name.split()) == 1: await naptha.hub.delete_kb(args.kb_name) elif len(args.kb_name.split()) == 1: @@ -952,7 +960,7 @@ async def main(): else: parsed_params = None - await run(naptha, args.agent, user_id, parsed_params, args.worker_node_urls, args.environment_node_urls, args.file, args.personas_urls) + await run(naptha, args.agent, user_id, parsed_params, args.worker_node_urls, args.environment_node_urls, args.kb_node_urls, args.file, args.personas_urls) elif args.command == "inference": request = ChatCompletionRequest( messages=[{"role": "user", "content": args.prompt}], diff --git a/naptha_sdk/schemas.py b/naptha_sdk/schemas.py index a553e28..438298b 100644 --- a/naptha_sdk/schemas.py +++ b/naptha_sdk/schemas.py @@ -47,12 +47,19 @@ class DataGenerationConfig(BaseModel): save_inputs: Optional[bool] = None save_inputs_location: Optional[str] = None +class KBDeployment(BaseModel): + name: Optional[str] = "kb_deployment" + module: Optional[Dict] = None + kb_node_url: Optional[str] = "http://localhost:7001" + kb_config: Optional[Dict] = None + class AgentDeployment(BaseModel): name: Optional[str] = "agent_deployment" module: Optional[Dict] = None worker_node_url: Optional[str] = None agent_config: Optional[AgentConfig] = AgentConfig() data_generation_config: Optional[DataGenerationConfig] = DataGenerationConfig() + kb_deployments: Optional[List[KBDeployment]] = None class EnvironmentDeployment(BaseModel): name: Optional[str] = "environment_deployment" @@ -60,12 +67,6 @@ class EnvironmentDeployment(BaseModel): environment_node_url: str environment_config: Optional[EnvironmentConfig] = EnvironmentConfig() -class KBDeployment(BaseModel): - name: Optional[str] = "kb_deployment" - module: Optional[Dict] = None - kb_node_url: Optional[str] = "http://localhost:7001" - kb_config: Optional[Dict] = None - class OrchestratorDeployment(BaseModel): name: Optional[str] = "orchestrator_deployment" module: Optional[Dict] = None From 04e92e8007fc0670bd942ca5fd3dbaa05e7b4aae Mon Sep 17 00:00:00 2001 From: Richard Blythman Date: Thu, 19 Dec 2024 23:52:01 +0000 Subject: [PATCH 21/21] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7c7cf1a..f4c3375 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ naptha run kb:wikipedia_kb -p "mode='query' query='Socrates'" ### Run an Agent that interacts with the Knowledge Base ```bash -naptha run agent:wikipedia_agent -p "query='socrates' question='Who is Socrates?'" --kb_node_urls "http://localhost:7001" +naptha run agent:wikipedia_agent -p "query='Socrates' question='Who is Socrates?'" --kb_node_urls "http://localhost:7001" ``` ## Personas