add ssh remote connection & validator installation for sn12

app/src/auto_validator/core/admin.py

@@ -32,10 +32,9 @@ class UploadedFileAdmin(admin.ModelAdmin):
 @admin.register(Subnet)
 class SubnetAdmin(admin.ModelAdmin):
     list_display = (
-        "name",
-        "description",
-        "mainnet_id",
-        "testnet_id",
+        "codename",
+        "mainnet_netuid",
+        "testnet_netuid",
         "owner_nick",
         "registered_networks",
     )

app/src/auto_validator/core/api.py

@@ -1,13 +1,24 @@
-from rest_framework import mixins, parsers, routers, viewsets
+import logging
+import pathlib
+
+from django.conf import settings
+from rest_framework import mixins, parsers, routers, status, viewsets
 from rest_framework.exceptions import AuthenticationFailed
 from rest_framework.permissions import AllowAny
+from rest_framework.response import Response
 
 from auto_validator.core.models import Hotkey, Server, UploadedFile, ValidatorInstance
 from auto_validator.core.serializers import UploadedFileSerializer
 from auto_validator.core.utils.utils import get_user_ip
 
 from .authentication import HotkeyAuthentication
 from .utils.bot import trigger_bot_send_message
+from .utils.utils import get_dumper_commands
+
+SUBNETS_CONFIG_PATH = pathlib.Path(settings.LOCAL_SUBNETS_SCRIPTS_PATH) / "subnets.yaml"
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
 
 
 class FilesViewSet(mixins.CreateModelMixin, mixins.ListModelMixin, viewsets.GenericViewSet):
@@ -50,6 +61,24 @@ def perform_create(self, serializer):
         )
 
 
+class DumperCommandsViewSet(viewsets.ViewSet):
+    parser_classes = [parsers.MultiPartParser]
+    permission_classes = [AllowAny]
+
+    def list(self, request):
+        subnet_identifier = request.headers.get("SubnetID")
+        if not subnet_identifier:
+            return Response({"error": "SubnetID is required"}, status=status.HTTP_400_BAD_REQUEST)
+
+        dumper_commands = get_dumper_commands(subnet_identifier, SUBNETS_CONFIG_PATH)
+        if dumper_commands is not None:
+            logger.info("SubnetID: %s, dumper_commands: %s", subnet_identifier, dumper_commands)
+            return Response(dumper_commands)
+        else:
+            logger.error("SubnetID: %s not found", subnet_identifier)
+            return Response({"error": "SubnetID not found"}, status=status.HTTP_404_NOT_FOUND)
+
+
 class APIRootView(routers.DefaultRouter.APIRootView):
     description = "api-root"
 
@@ -60,3 +89,4 @@ class APIRouter(routers.DefaultRouter):
 
 router = APIRouter()
 router.register(r"files", FilesViewSet, basename="file")
+router.register(r"commands", DumperCommandsViewSet, basename="commands")

app/src/auto_validator/core/migrations/0012_rename_hw_requirements_subnet_hardware_description_and_more.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.2.16 on 2024-10-08 01:54
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0011_alter_subnet_hw_requirements"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="subnet",
+            old_name="hw_requirements",
+            new_name="hardware_description",
+        ),
+        migrations.RenameField(
+            model_name="subnet",
+            old_name="mainnet_id",
+            new_name="mainnet_netuid",
+        ),
+        migrations.RenameField(
+            model_name="subnet",
+            old_name="maintainers_ids",
+            new_name="maintainer_discord_ids",
+        ),
+        migrations.RenameField(
+            model_name="subnet",
+            old_name="owner_id",
+            new_name="owner_discord_id",
+        ),
+        migrations.RenameField(
+            model_name="subnet",
+            old_name="testnet_id",
+            new_name="testnet_netuid",
+        ),
+        migrations.AddField(
+            model_name="subnet",
+            name="allowed_secrets",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=255),
+                blank=True,
+                null=True,
+                size=None,
+            ),
+        ),
+        migrations.AddField(
+            model_name="subnet",
+            name="dumper_commands",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(max_length=255),
+                blank=True,
+                null=True,
+                size=None,
+            ),
+        ),
+    ]

app/src/auto_validator/core/models.py

@@ -49,13 +49,15 @@ class Subnet(models.Model):
     description = models.TextField(null=True, blank=True)
     operators = models.ManyToManyField("Operator", related_name="subnets", blank=True)
     codename = models.CharField(max_length=255, null=True, blank=True)
-    mainnet_id = models.IntegerField(null=True, blank=True)
-    testnet_id = models.IntegerField(null=True, blank=True)
+    mainnet_netuid = models.IntegerField(null=True, blank=True)
+    testnet_netuid = models.IntegerField(null=True, blank=True)
     owner_nick = models.CharField(max_length=255, null=True, blank=True)
-    owner_id = models.CharField(max_length=255, null=True, blank=True)
-    maintainers_ids = ArrayField(models.CharField(max_length=255), null=True, blank=True)
+    owner_discord_id = models.CharField(max_length=255, null=True, blank=True)
+    maintainer_discord_ids = ArrayField(models.CharField(max_length=255), null=True, blank=True)
     github_repo = models.CharField(max_length=255, null=True, blank=True)
-    hw_requirements = models.TextField(max_length=4095, null=True, blank=True)
+    hardware_description = models.TextField(max_length=4095, null=True, blank=True)
+    allowed_secrets = ArrayField(models.CharField(max_length=255), null=True, blank=True)
+    dumper_commands = ArrayField(models.CharField(max_length=255), null=True, blank=True)
 
     def registered_networks(self):
         mainnet_slots = self.slots.filter(

app/src/auto_validator/core/tasks.py

@@ -1,13 +1,20 @@
-import bittensor as bt
+import os
+import shutil
+
+import bittensor as bt  # type: ignore
 import structlog
-from celery import shared_task
-from celery.utils.log import get_task_logger
+from celery import shared_task  # type: ignore
+from celery.utils.log import get_task_logger  # type: ignore
 from django.conf import settings
+from git import GitCommandError, Repo
 
 from auto_validator.celery import app
 
 from .models import SubnetSlot, ValidatorInstance
 
+GITHUB_SUBNETS_SCRIPTS_PATH = settings.GITHUB_SUBNETS_SCRIPTS_PATH
+LOCAL_SUBNETS_SCRIPTS_PATH = settings.LOCAL_SUBNETS_SCRIPTS_PATH
+
 logger = structlog.wrap_logger(get_task_logger(__name__))
 
 
@@ -53,3 +60,26 @@ def update_validator_status_for_slot(slot_id):
 
 def fetch_last_updated_from_metagraph(metagraph, public_key):
     return metagraph.last_update[metagraph.hotkeys.index(public_key)]
+
+
+@app.task
+def schedule_fetch_subnet_scripts():
+    fetch_subnet_scripts.delay()
+
+
+@shared_task
+def fetch_subnet_scripts():
+    logger.info("Fetching subnet scripts")
+    try:
+        # Clone the subnet scripts repository using GitPython
+        if os.path.exists(LOCAL_SUBNETS_SCRIPTS_PATH):
+            # If the directory already exists, remove it
+            shutil.rmtree(LOCAL_SUBNETS_SCRIPTS_PATH)
+
+        Repo.clone_from(GITHUB_SUBNETS_SCRIPTS_PATH, LOCAL_SUBNETS_SCRIPTS_PATH)
+    except GitCommandError as e:
+        logger.error(f"Error while cloning the repository: {e}")
+        return
+
+    logger.info("Successfully fetched subnet scripts")
+    return

app/src/auto_validator/core/tests/conftest.py

@@ -1,7 +1,6 @@
 from collections.abc import Generator
 
 import bittensor as bt
-import pexpect
 import pytest
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APIClient
@@ -86,42 +85,11 @@ def __eq__(self, other):
 def wallet():
     coldkey_name = "auto-validator7"
     hotkey_name = "testhotkey7"
-    command1 = f"btcli wallet new_coldkey --wallet.name {coldkey_name}"
-    command2 = f"btcli wallet new_hotkey --wallet.name {coldkey_name} --wallet.hotkey {hotkey_name}"
-    has_coldkey = False
-    has_hotkey = False
-    password = "your_password_here"
-
-    try:
-        wallet = bt.wallet(name=coldkey_name, hotkey=hotkey_name)
-        wallet.coldkeypub  # make sure wallet has coldkey file, if not, it will raise an exception
-        has_coldkey = True
-        wallet.hotkey  # make sure wallet has hotkey file, if not, it will raise an exception
-        has_hotkey = True
-    except bt.KeyFileError:
-        if not has_coldkey:
-            process = pexpect.spawn(command1, timeout=30)
-            try:
-                process.expect("Specify password for key encryption:")
-                process.sendline(password)
-
-                process.expect("Retype your password:")
-                process.sendline(password)
-
-                process.expect(pexpect.EOF)
-            except pexpect.TIMEOUT:
-                print("Timeout occurred while creating coldkey.")
-            finally:
-                process.close()
-
-        if not has_hotkey:
-            process = pexpect.spawn(command2, timeout=30)
-            try:
-                process.expect(pexpect.EOF)
-            except pexpect.TIMEOUT:
-                print("Timeout occurred while creating hotkey.")
-            finally:
-                process.close()
-        wallet = bt.wallet(name=coldkey_name, hotkey=hotkey_name)
+
+    wallet = bt.Wallet(name=coldkey_name, hotkey=hotkey_name, path=".bittensor/wallets")
+    if not wallet.coldkey_file.exists_on_device():
+        wallet.create_new_coldkey(overwrite=True, use_password=False)
+    if not wallet.hotkey_file.exists_on_device():
+        wallet.create_new_hotkey(overwrite=True, use_password=False)
 
     return wallet

app/src/auto_validator/core/utils/childhotkey.py

@@ -0,0 +1,121 @@
+import logging
+import sys
+from io import StringIO
+
+from bittensor import Wallet
+from bittensor_cli import CLIManager
+
+from ..models import Hotkey
+
+
+class CLIManagerWrapper:
+    def __init__(self):
+        self.cli_manager = CLIManager()
+
+    def __enter__(self):
+        sys.stdin = StringIO("y\n")
+        return self
+
+    def __exit__(self, type, value, traceback):
+        sys.stdin = sys.__stdin__
+
+    def stake_revoke_children(self, *args, **kwargs):
+        result = self.cli_manager.stake_revoke_children(*args, **kwargs)
+        return result
+
+    def stake_get_children(self, *args, **kwargs):
+        result = self.cli_manager.stake_get_children(*args, **kwargs)
+        return result
+
+    def stake_set_children(self, *args, **kwargs):
+        result = self.cli_manager.stake_set_children(*args, **kwargs)
+        return result
+
+
+class ChildHotkey:
+    def __init__(
+        self, parent_wallet_name: str, parent_hotkey_name: str, parent_wallet_path: str = "~/.bittensor/wallets"
+    ):
+        self.logger = logging.getLogger(__name__)
+        self.parent_wallet_name = parent_wallet_name
+        self.parent_hotkey_name = parent_hotkey_name
+        self.parent_wallet_path = parent_wallet_path
+
+    def connect_to_parent_wallet(self):
+        self.parent_wallet = Wallet(name=self.parent_wallet_name, hotkey=self.parent_hotkey_name)
+        if not self.parent_wallet.coldkey_file.exists_on_device():
+            raise ValueError("Coldkey file for parent wallet %s not found.", self.parent_wallet_name)
+        if not self.parent_wallet.hotkey_file.exists_on_device():
+            raise ValueError("Hotkey file for parent wallet %s not found.", self.parent_wallet_name)
+
+    def __enter__(self):
+        self.connect_to_parent_wallet()
+        return self
+
+    def create_new_child_hotkey(
+        self,
+        network: str,
+        netuid: int,
+        child_wallet_name: str,
+        child_hotkey_name: str,
+        proportion: float = 1,
+    ) -> str:
+        child_wallet = Wallet(name=child_wallet_name, hotkey=child_hotkey_name)
+        if not child_wallet.coldkey_file.exists_on_device():
+            child_wallet.create_new_coldkey(overwrite=False, use_password=False)
+        if not child_wallet.hotkey_file.exists_on_device():
+            child_wallet.create_new_hotkey(overwrite=False, use_password=False)
+
+        cli_manager = CLIManagerWrapper()
+
+        with CLIManagerWrapper() as cli_manager:
+            cli_manager.stake_set_children(
+                wallet_name=self.parent_wallet_name,
+                wallet_hotkey=self.parent_hotkey_name,
+                wallet_path=self.parent_wallet_path,
+                network=network,
+                netuid=netuid,
+                all_netuids=False,
+                children=[child_wallet.hotkey.ss58_address],
+                proportions=[proportion],
+                quiet=True,
+                verbose=False,
+                wait_for_finalization=True,
+                wait_for_inclusion=True,
+            )
+        Hotkey.objects.create(hotkey=child_wallet.hotkey.ss58_address)
+        return child_wallet.hotkey.ss58_address
+
+    def get_child_hotkeys(self, network: str, netuid: int):
+        with CLIManagerWrapper() as cli_manager:
+            result = cli_manager.stake_get_children(
+                wallet_name=self.parent_wallet_name,
+                wallet_hotkey=self.parent_hotkey_name,
+                wallet_path=self.parent_wallet_path,
+                network=network,
+                netuid=netuid,
+                all_netuids=False,
+                quiet=True,
+                verbose=False,
+            )
+        return result
+
+    def revoke_child_hotkeys(
+        self,
+        network: str,
+        netuid: int,
+    ) -> bool:
+        with CLIManagerWrapper() as cli_manager:
+            cli_manager.stake_revoke_children(
+                wallet_name=self.parent_wallet_name,
+                wallet_hotkey=self.parent_hotkey_name,
+                wallet_path=self.parent_wallet_path,
+                network=network,
+                netuid=netuid,
+                all_netuids=False,
+                quiet=True,
+                verbose=False,
+                wait_for_finalization=True,
+                wait_for_inclusion=True,
+            )
+        return True