Merge branch 'main' of https://github.com/believethehype/nostrdvm

main.py

@@ -5,7 +5,7 @@
 from nostr_dvm.bot import Bot
 from nostr_dvm.tasks import videogeneration_replicate_svd, imagegeneration_replicate_sdxl, textgeneration_llmlite, \
     trending_notes_nostrband, discovery_inactive_follows, translation_google, textextraction_pdf, \
-    translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle
+    translation_libretranslate, textextraction_google, convert_media, imagegeneration_openai_dalle, texttospeech
 from nostr_dvm.utils.admin_utils import AdminConfig
 from nostr_dvm.utils.backend_utils import keep_alive
 from nostr_dvm.utils.definitions import EventDefinitions
@@ -133,6 +133,11 @@ def playground():
     ollama = textgeneration_llmlite.build_example("LLM", "llmlite", admin_config)
     bot_config.SUPPORTED_DVMS.append(ollama)
     ollama.run()
+
+    tts = texttospeech.build_example("Text To Speech Test", "tts", admin_config)
+    bot_config.SUPPORTED_DVMS.append(tts)
+    tts.run()
+
     # Run the bot
     Bot(bot_config)
     # Keep the main function alive for libraries that require it, like openai

nostr_dvm/dvm.py

@@ -1,5 +1,9 @@
+import importlib
 import json
+import os
+import subprocess
 from datetime import timedelta
+from pathlib import Path
 
 from nostr_sdk import PublicKey, Keys, Client, Tag, Event, EventBuilder, Filter, HandleNotification, Timestamp, \
     init_logger, LogLevel, Options, nip04_encrypt
@@ -323,7 +327,8 @@ def check_and_return_event(data, original_event: Event):
                         except Exception as e:
                             # Zapping back by error in post-processing is a risk for the DVM because work has been done,
                             # but maybe something with parsing/uploading failed. Try to avoid errors here as good as possible
-                            send_job_status_reaction(original_event, "error", content="Error in Post-processing: " + str(e),
+                            send_job_status_reaction(original_event, "error",
+                                                     content="Error in Post-processing: " + str(e),
                                                      dvm_config=self.dvm_config,
                                                      )
                             if amount > 0 and self.dvm_config.LNBITS_ADMIN_KEY != "":
@@ -466,11 +471,28 @@ def do_work(job_event, amount):
                 for dvm in self.dvm_config.SUPPORTED_DVMS:
                     try:
                         if task == dvm.TASK:
-                            request_form = dvm.create_request_from_nostr_event(job_event, self.client,
-                                                                               self.dvm_config)
-                            result = dvm.process(request_form)
+
+                            request_form = dvm.create_request_from_nostr_event(job_event, self.client, self.dvm_config)
+
+                            if dvm_config.USE_OWN_VENV:
+                                python_bin = (r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0]
+                                              + "/bin/python")
+                                retcode = subprocess.call([python_bin, dvm_config.SCRIPT,
+                                                '--request', json.dumps(request_form),
+                                                '--identifier', dvm_config.IDENTIFIER,
+                                                '--output', 'output.txt'])
+                                print("Finished processing, loading data..")
+
+                                with open(os.path.abspath('output.txt')) as f:
+                                    result = f.readlines()[0]
+                                    print(result)
+                                #f.close()
+                                os.remove(os.path.abspath('output.txt'))
+                            else: #Some components might have issues with running code in otuside venv.
+                                  # We install locally in these cases for now
+                                result = dvm.process(request_form)
                             try:
-                                post_processed = dvm.post_process(result, job_event)
+                                post_processed = dvm.post_process(str(result), job_event)
                                 send_nostr_reply_event(post_processed, job_event.as_json())
                             except Exception as e:
                                 send_job_status_reaction(job_event, "error", content=str(e),
@@ -495,7 +517,6 @@ def do_work(job_event, amount):
                             except Exception as e:
                                 print(e)
 
-
                         return
 
         self.client.handle_notifications(NotificationHandler())

nostr_dvm/interfaces/dvmtaskinterface.py

@@ -1,18 +1,16 @@
 import json
 import os
 import subprocess
+from subprocess import run
 import sys
 from threading import Thread
-
+from venv import create
 from nostr_sdk import Keys
-
 from nostr_dvm.dvm import DVM
 from nostr_dvm.utils.admin_utils import AdminConfig
 from nostr_dvm.utils.dvmconfig import DVMConfig
-from nostr_dvm.utils.nip89_utils import NIP89Config
-from nostr_dvm.utils.nostr_utils import check_and_set_private_key
+from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
 from nostr_dvm.utils.output_utils import post_process_result
-from nostr_dvm.utils.zap_utils import check_and_set_ln_bits_keys
 
 
 class DVMTaskInterface:
@@ -34,7 +32,7 @@ def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config, admin_
                  options=None, task=None):
         self.init(name, dvm_config, admin_config, nip89config, task)
         self.options = options
-        self.install_dependencies(self.dependencies)
+        self.install_dependencies(dvm_config)
 
     def init(self, name, dvm_config, admin_config=None, nip89config=None, task=None):
         self.NAME = name
@@ -58,6 +56,25 @@ def init(self, name, dvm_config, admin_config=None, nip89config=None, task=None)
         self.dvm_config = dvm_config
         self.admin_config = admin_config
 
+    def install_dependencies(self, dvm_config):
+        if dvm_config.SCRIPT != "":
+            if self.dvm_config.USE_OWN_VENV:
+                dir = r'cache/venvs/' + os.path.basename(dvm_config.SCRIPT).split(".py")[0]
+                if not os.path.isdir(dir):
+                    print(dir)
+                    create(dir, with_pip=True, upgrade_deps=True)
+                    for (module, package) in self.dependencies:
+                        print("Installing Venv Module: " + module)
+                        run(["bin/pip", "install", package], cwd=dir)
+            else:
+                for module, package in self.dependencies:
+                    if module != "nostr-dvm":
+                        try:
+                            __import__(module)
+                        except ImportError:
+                            print("Installing global Module: " + module)
+                            subprocess.check_call([sys.executable, "-m", "pip", "install", package])
+
     def run(self):
         nostr_dvm_thread = Thread(target=self.DVM, args=[self.dvm_config, self.admin_config])
         nostr_dvm_thread.start()
@@ -87,16 +104,6 @@ def post_process(self, result, event):
         """Post-process the data and return the result Use default function, if not overwritten"""
         return post_process_result(result, event)
 
-    def install_dependencies(self, packages):
-        import pip
-        for module, package in packages:
-            try:
-                __import__(module)
-            except ImportError:
-                subprocess.check_call([sys.executable, "-m", "pip", "install", package])
-
-
-
     @staticmethod
     def set_options(request_form):
         print("Setting options...")
@@ -105,3 +112,19 @@ def set_options(request_form):
             opts = json.loads(request_form["options"])
             print(opts)
         return dict(opts)
+
+    @staticmethod
+    def process_args():
+        import argparse
+        parser = argparse.ArgumentParser()
+        parser.add_argument('--request', dest='request')
+        parser.add_argument('--identifier', dest='identifier')
+        parser.add_argument('--output', dest='output')
+        args = parser.parse_args()
+        return args
+
+    @staticmethod
+    def write_output(result, output):
+        with open(os.path.abspath(output), 'w') as f:
+            f.write(result)
+        # f.close()

nostr_dvm/tasks/advanced_search.py

@@ -1,14 +1,10 @@
 import json
 import os
 from datetime import timedelta
-from pathlib import Path
-
-import dotenv
 from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options, SecretKey
 
 from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
 from nostr_dvm.utils.admin_utils import AdminConfig
-from nostr_dvm.utils.backend_utils import keep_alive
 from nostr_dvm.utils.definitions import EventDefinitions
 from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config
 from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
@@ -27,9 +23,11 @@ class AdvancedSearch(DVMTaskInterface):
     TASK: str = "search-content"
     FIX_COST: float = 0
     dvm_config: DVMConfig
+    dependencies = [("nostr-dvm", "nostr-dvm")]
 
     def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
                  admin_config: AdminConfig = None, options=None):
+        dvm_config.SCRIPT = os.path.abspath(__file__)
         super().__init__(name, dvm_config, nip89config, admin_config, options)
 
     def is_input_supported(self, tags):
@@ -177,19 +175,12 @@ def build_example(name, identifier, admin_config):
                                    admin_config=admin_config)
 
 
-if __name__ == '__main__':
-    env_path = Path('.env')
-    if env_path.is_file():
-        print(f'loading environment from {env_path.resolve()}')
-        dotenv.load_dotenv(env_path, verbose=True, override=True)
-    else:
-        raise FileNotFoundError(f'.env file not found at {env_path} ')
-
-    admin_config = AdminConfig()
-    admin_config.REBROADCAST_NIP89 = False
-    admin_config.UPDATE_PROFILE = False
+def process_venv():
+    args = DVMTaskInterface.process_args()
+    dvm_config = build_default_config(args.identifier)
+    dvm = AdvancedSearch(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None)
+    result = dvm.process(json.loads(args.request))
+    DVMTaskInterface.write_output(result, args.output)
 
-    dvm = build_example("Advanced Nostr Search", "discovery_content_search", admin_config)
-    dvm.run()
-
-    keep_alive()
+if __name__ == '__main__':
+    process_venv()

nostr_dvm/tasks/convert_media.py

@@ -1,11 +1,7 @@
 import json
-from pathlib import Path
-
-import dotenv
-
+import os
 from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
 from nostr_dvm.utils.admin_utils import AdminConfig
-from nostr_dvm.utils.backend_utils import keep_alive
 from nostr_dvm.utils.definitions import EventDefinitions
 from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config
 from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
@@ -30,6 +26,7 @@ class MediaConverter(DVMTaskInterface):
 
     def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
                  admin_config: AdminConfig = None, options=None):
+        dvm_config.SCRIPT = os.path.abspath(__file__)
         super().__init__(name, dvm_config, nip89config, admin_config, options)
 
     def is_input_supported(self, tags):
@@ -100,26 +97,18 @@ def build_example(name, identifier, admin_config):
     }
 
     nip89config = NIP89Config()
-    nip89config.DTAG = check_and_set_d_tag(identifier, name, dvm_config.PRIVATE_KEY, nip89info["image"])
-    nip89config.CONTENT = json.dumps(nip89info)
+
 
     return MediaConverter(name=name, dvm_config=dvm_config, nip89config=nip89config,
                           admin_config=admin_config)
 
+def process_venv():
+    args = DVMTaskInterface.process_args()
+    dvm_config = build_default_config(args.identifier)
+    dvm = MediaConverter(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None)
+    result = dvm.process(json.loads(args.request))
+    DVMTaskInterface.write_output(result, args.output)
 
-if __name__ == '__main__':
-    env_path = Path('.env')
-    if env_path.is_file():
-        print(f'loading environment from {env_path.resolve()}')
-        dotenv.load_dotenv(env_path, verbose=True, override=True)
-    else:
-        raise FileNotFoundError(f'.env file not found at {env_path} ')
 
-    admin_config = AdminConfig()
-    admin_config.REBROADCAST_NIP89 = False
-    admin_config.UPDATE_PROFILE = False
-
-    dvm = build_example("Media Bringer", "media_converter", admin_config)
-    dvm.run()
-
-    keep_alive()
+if __name__ == '__main__':
+    process_venv()

nostr_dvm/tasks/discovery_inactive_follows.py

@@ -1,15 +1,12 @@
 import json
 import os
 from datetime import timedelta
-from pathlib import Path
 from threading import Thread
 
-import dotenv
 from nostr_sdk import Client, Timestamp, PublicKey, Tag, Keys, Options
 
 from nostr_dvm.interfaces.dvmtaskinterface import DVMTaskInterface
 from nostr_dvm.utils.admin_utils import AdminConfig
-from nostr_dvm.utils.backend_utils import keep_alive
 from nostr_dvm.utils.definitions import EventDefinitions
 from nostr_dvm.utils.dvmconfig import DVMConfig, build_default_config
 from nostr_dvm.utils.nip89_utils import NIP89Config, check_and_set_d_tag
@@ -33,6 +30,7 @@ class DiscoverInactiveFollows(DVMTaskInterface):
 
     def __init__(self, name, dvm_config: DVMConfig, nip89config: NIP89Config,
                  admin_config: AdminConfig = None, options=None):
+        dvm_config.SCRIPT = os.path.abspath(__file__)
         super().__init__(name, dvm_config, nip89config, admin_config, options)
 
     def is_input_supported(self, tags):
@@ -198,19 +196,12 @@ def build_example(name, identifier, admin_config):
     return DiscoverInactiveFollows(name=name, dvm_config=dvm_config, nip89config=nip89config,
                                    admin_config=admin_config)
 
+def process_venv():
+    args = DVMTaskInterface.process_args()
+    dvm_config = build_default_config(args.identifier)
+    dvm = DiscoverInactiveFollows(name="", dvm_config=dvm_config, nip89config=NIP89Config(), admin_config=None)
+    result = dvm.process(json.loads(args.request))
+    DVMTaskInterface.write_output(result, args.output)
 
 if __name__ == '__main__':
-    env_path = Path('.env')
-    if env_path.is_file():
-        print(f'loading environment from {env_path.resolve()}')
-        dotenv.load_dotenv(env_path, verbose=True, override=True)
-    else:
-        raise FileNotFoundError(f'.env file not found at {env_path} ')
-
-    admin_config = AdminConfig()
-    admin_config.REBROADCAST_NIP89 = False
-    admin_config.UPDATE_PROFILE = False
-    dvm = build_example("Bygones", "discovery_inactive_follows", admin_config)
-    dvm.run()
-
-    keep_alive()
+    process_venv()