Fixes issue #63: Ability to allow publishing from module repo

naptha_sdk/cli.py

@@ -975,7 +975,9 @@ async def main():
 
     # Publish command
     publish_parser = subparsers.add_parser("publish", help="Publish agents.")
-
+    publish_parser.add_argument("-d", "--decorator", help="Publish module via decorator", action="store_true")
+    publish_parser.add_argument("-r", "--register", help="Register agents with hub", action="store_true")
+
     async with naptha as naptha:
         args = parser.parse_args()
         args = _parse_str_args(args)
@@ -1256,7 +1258,7 @@ async def main():
             elif args.command == "storage":
                 await storage_interaction(naptha, args.storage_type, args.command, args.path, args.data, args.schema, args.options)
             elif args.command == "publish":
-                await naptha.publish_agents()
+                await naptha.publish_agents(args.decorator, args.register)
         else:
             parser.print_help()
 

naptha_sdk/client/hub.py

@@ -212,6 +212,23 @@ async def list_kbs(self, kb_name=None) -> List:
             kb = await self.surrealdb.query("SELECT * FROM kb WHERE name=$kb_name;", {"kb_name": kb_name})
             return kb[0]['result']
 
+    async def list_modules(self, module_type, module_name) -> List:
+        if module_type == 'agent':
+            modules = await self.surrealdb.query("SELECT * FROM agent WHERE id=$module_name;", {"module_name": module_name})
+        elif module_type == 'tool':
+            modules = await self.surrealdb.query("SELECT * FROM tool WHERE id=$module_name;", {"module_name": module_name})
+        elif module_type == 'orchestrator':
+            modules = await self.surrealdb.query("SELECT * FROM orchestrator WHERE id=$module_name;", {"module_name": module_name})
+        elif module_type == 'environment':
+            modules = await self.surrealdb.query("SELECT * FROM environment WHERE id=$module_name;", {"module_name": module_name})
+        elif module_type == 'persona':
+            modules = await self.surrealdb.query("SELECT * FROM persona WHERE id=$module_name;", {"module_name": module_name})
+        elif module_type == 'memory':
+            modules = await self.surrealdb.query("SELECT * FROM memory WHERE id=$module_name;", {"module_name": module_name})
+        elif module_type == 'kb':
+            modules = await self.surrealdb.query("SELECT * FROM kb WHERE id=$module_name;", {"module_name": module_name})
+        return modules[0]['result']
+
     async def list_kb_content(self, kb_name: str) -> List:
         kb_content = await self.surrealdb.query("SELECT * FROM kb_content WHERE kb_id=$kb_id;", {"kb_id": f"kb:{kb_name}"})
         return kb_content[0]['result']
@@ -338,12 +355,14 @@ async def create_kb(self, kb_config: Dict) -> Tuple[bool, Optional[Dict]]:
     async def update_agent(self, agent_config: Dict) -> Tuple[bool, Optional[Dict]]:
         return await self.surrealdb.update("agent", agent_config)
 
-    async def create_or_update_agent(self, agent_config: Dict) -> Tuple[bool, Optional[Dict]]:
-        list_agents = await self.list_agents(agent_config.get('id'))
-        if not list_agents:
-            return await self.surrealdb.create("agent", agent_config)
+    async def create_or_update_module(self, module_type, module_config: Dict) -> Tuple[bool, Optional[Dict]]:
+        list_modules = await self.list_modules(module_type, module_config.get('id'))
+        if not list_modules:
+            logger.info(f"Module does not exist. Registering new module: {module_config.get('id')}")
+            return await self.surrealdb.create(module_type, module_config)
         else:
-            return await self.surrealdb.update(agent_config.pop('id'), agent_config)
+            logger.info(f"Module already exists. Updating existing module: {module_config.get('id')}")
+            return await self.surrealdb.update(module_config.pop('id'), module_config)
 
     async def close(self):
         """Close the database connection"""

naptha_sdk/client/naptha.py

@@ -1,11 +1,11 @@
 import asyncio
+from dotenv import load_dotenv
 import inspect
+import json
 import os
 import time
 from pathlib import Path
 
-from dotenv import load_dotenv
-
 from naptha_sdk.client.hub import Hub
 from naptha_sdk.client.node import UserClient
 from naptha_sdk.inference import InferenceClient
@@ -67,38 +67,50 @@ async def create_agent(self, name):
             else:
                 logger.error(f"Failed to create agent {name}")
 
-    async def publish_agents(self):
+    async def publish_agents(self, decorator = False, register = False):
         logger.info(f"Publishing Agent Packages...")
         start_time = time.time()
 
-        path = Path.cwd() / AGENT_DIR
-        agents = [item.name for item in path.iterdir() if item.is_dir()]
-
-        agent = agents[0]
-        for agent in agents:
-            git_add_commit(agent)
-            _, response = await publish_ipfs_package(agent)
-            logger.info(f"Published Agent: {agent}")
-
-            # Register agent with hub
-            async with self.hub:
-                _, _, user_id = await self.hub.signin(self.hub_username, os.getenv("HUB_PASSWORD"))
-                agent_config = {
-                    "id": f"agent:{agent}",
-                    "name": agent,
-                    "description": agent,
-                    "parameters": agent,
-                    "author": self.hub.user_id,
-                    "module_url": f'ipfs://{response["ipfs_hash"]}',
-                    "module_type": "package",
-                    "module_version": "0.1"
-                }
-                logger.info(f"Registering Agent {agent_config}")
-                agent = await self.hub.create_or_update_agent(agent_config)
+        if not decorator:
+            module_path = Path.cwd() 
+            modules = [module_path.name]
+            with open(module_path / module_path.name / 'configs/deployment.json', 'r') as f:
+                deployment = json.load(f)
+                module_type = deployment[0]['module']['module_type']
+        else:
+            path = Path.cwd() / AGENT_DIR
+            modules = [item.name for item in path.iterdir() if item.is_dir()]
+            module_type = 'agent'
+            for module in modules:
+                git_add_commit(module)
+
+        for module in modules:
+            _, response = await publish_ipfs_package(module, decorator)
+            logger.info(f"Storing {module_type} {module} on IPFS")
+            logger.info(f"IPFS Hash: {response['ipfs_hash']}. You can download it from http://provider.akash.pro:30584/ipfs/{response['ipfs_hash']}")
+
+            if register:
+                # Register module with hub
+                async with self.hub:
+                    _, _, user_id = await self.hub.signin(self.hub_username, os.getenv("HUB_PASSWORD"))
+                    module_config = {
+                        "id": f"{module_type}:{module}",
+                        "name": module,
+                        "description": module,
+                        "parameters": module,
+                        "author": self.hub.user_id,
+                        "module_url": f'ipfs://{response["ipfs_hash"]}',
+                        "module_type": module_type,
+                        "module_version": "v0.1",
+                        "module_entrypoint": "run.py",
+                        "execution_type": "package",
+                    }
+                    logger.info(f"Registering {module_type} {module} on Naptha Hub {module_config}")
+                    module = await self.hub.create_or_update_module(module_type, module_config)
 
         end_time = time.time()
         total_time = end_time - start_time
-        logger.info(f"Total time taken to publish {len(agents)} agents: {total_time:.2f} seconds")
+        logger.info(f"Total time taken to publish {len(modules)} modules: {total_time:.2f} seconds")
 
     def build(self):
         asyncio.run(self.build_agents())

naptha_sdk/module_manager.py

@@ -15,6 +15,7 @@
 import tomlkit
 import yaml
 import zipfile
+import fnmatch
 
 load_dotenv()
 logger = get_logger(__name__)
@@ -283,9 +284,14 @@ async def write_to_ipfs(file_path):
         logger.error(f"Traceback: {traceback.format_exc()}")
         return (500, {"message": f"Error writing file to IPFS: {e}"})
 
-async def publish_ipfs_package(agent_name):
+async def publish_ipfs_package(agent_name, decorator = False):
     package_path = f"{AGENT_DIR}/{agent_name}"
-    output_zip_file = zip_dir(package_path)
+
+    if not decorator:
+        output_zip_file = zip_dir_with_gitignore(Path.cwd())
+    else:
+        output_zip_file = zip_dir(package_path)
+
     success, response = await write_to_ipfs(output_zip_file)
     logger.info(f"Response: {response}")
     return success, response
@@ -371,4 +377,42 @@ async def load_persona(persona_module):
     # input_schema = load_input_schema(repo_name)
     return persona_data
 
-
+
+
+def read_gitignore(directory):
+    gitignore_path = os.path.join(directory, '.gitignore')
+
+    if not os.path.exists(gitignore_path):
+        logger.info(f"No .gitignore file found in {directory}")
+        return []
+
+    with open(gitignore_path, 'r') as file:
+        lines = file.readlines()
+
+    ignored_files = [line.strip() for line in lines if line.strip() and not line.startswith('#')]
+    return ignored_files
+
+def zip_dir_with_gitignore(directory_path):
+    ignored_files = read_gitignore(directory_path)
+    output_zip_file = f"./{os.path.basename(directory_path)}.zip"
+
+    # Convert patterns in .gitignore to absolute paths for comparison
+    ignored_patterns = [os.path.join(directory_path, pattern) for pattern in ignored_files]
+
+    with zipfile.ZipFile(output_zip_file, "w", zipfile.ZIP_DEFLATED) as zip_file:
+        for root, dirs, files in os.walk(directory_path):
+            dirs = [d for d in dirs if not any(fnmatch.fnmatch(os.path.join(root, d), pattern) for pattern in ignored_patterns)]
+
+            for file in files:
+                file_path = os.path.join(root, file)
+
+                if any(fnmatch.fnmatch(file_path, pattern) for pattern in ignored_patterns):
+                    continue
+
+                if file == output_zip_file.split('/')[1]:
+                    continue
+
+                zip_file.write(file_path, os.path.relpath(file_path, directory_path))
+
+    logger.info(f"Zipped directory '{directory_path}' to '{output_zip_file}'")
+    return output_zip_file