feat(engine): Add and update git and ssh modules (#754)

frontend/src/client/schemas.gen.ts

@@ -2353,6 +2353,7 @@ export const $Role = {
         "tracecat-schedule-runner",
         "tracecat-service",
         "tracecat-executor",
+        "tracecat-bootstrap",
       ],
       title: "Service Id",
     },

frontend/src/client/types.gen.ts

@@ -781,6 +781,7 @@ export type Role = {
     | "tracecat-schedule-runner"
     | "tracecat-service"
     | "tracecat-executor"
+    | "tracecat-bootstrap"
 }
 
 export type type2 = "user" | "service"
@@ -792,6 +793,7 @@ export type service_id =
   | "tracecat-schedule-runner"
   | "tracecat-service"
   | "tracecat-executor"
+  | "tracecat-bootstrap"
 
 /**
  * This object contains all the information needed to execute an action.

tests/unit/test_registry.py

@@ -154,17 +154,17 @@ async def test_registry_async_function_can_be_called(mock_package):
                 host="github.com",
                 org="org",
                 repo="repo",
-                branch="main",
+                ref=None,
             ),
         ),
-        # GitHub (with branch)
+        # GitHub (with branch/sha)
         (
-            "git+ssh://[email protected]/org/repo@branch",
+            "git+ssh://[email protected]/org/repo@branchOrSHAOrTag",
             GitUrl(
                 host="github.com",
                 org="org",
                 repo="repo",
-                branch="branch",
+                ref="branchOrSHAOrTag",
             ),
         ),
         # GitLab
@@ -174,7 +174,7 @@ async def test_registry_async_function_can_be_called(mock_package):
                 host="gitlab.com",
                 org="org",
                 repo="repo",
-                branch="main",
+                ref=None,
             ),
         ),
         # GitLab (with branch)
@@ -184,7 +184,7 @@ async def test_registry_async_function_can_be_called(mock_package):
                 host="gitlab.com",
                 org="org",
                 repo="repo",
-                branch="branch",
+                ref="branch",
             ),
         ),
     ],

tracecat/api/common.py

@@ -28,7 +28,7 @@ def bootstrap_role():
     return Role(
         type="service",
         access_level=AccessLevel.ADMIN,
-        service_id="tracecat-api",
+        service_id="tracecat-bootstrap",
     )
 
 

tracecat/dsl/scheduler.py

@@ -148,7 +148,7 @@ async def _queue_tasks(
         )
         async with asyncio.TaskGroup() as tg:
             for next_ref, edge_type in next_tasks:
-                self.logger.warning("Processing next task", ref=ref, next_ref=next_ref)
+                self.logger.debug("Processing next task", ref=ref, next_ref=next_ref)
                 edge = DSLEdge(src=ref, dst=next_ref, type=edge_type)
                 if unreachable and edge in unreachable:
                     self._mark_edge(edge, EdgeMarker.SKIPPED)
@@ -272,7 +272,7 @@ def _is_reachable(self, task: ActionStatement) -> bool:
             # Root nodes are always reachable
             return True
         elif n_deps == 1:
-            logger.warning("Task has only 1 dependency", task=task)
+            logger.debug("Task has only 1 dependency", task=task)
             # If there's only 1 dependency, the node is reachable only if the
             # dependency was successful ignoring the join strategy.
             dep_ref = task.depends_on[0]

tracecat/git.py

@@ -1,21 +1,33 @@
 import asyncio
 import re
 from dataclasses import dataclass
+from typing import cast
 
+from tracecat import config
+from tracecat.contexts import ctx_role
 from tracecat.logger import logger
+from tracecat.registry.repositories.service import RegistryReposService
+from tracecat.settings.service import get_setting
 from tracecat.ssh import SshEnv
+from tracecat.types.auth import Role
+from tracecat.types.exceptions import TracecatSettingsError
 
 GIT_SSH_URL_REGEX = re.compile(
-    r"^git\+ssh://git@(?P<host>[^/]+)/(?P<org>[^/]+)/(?P<repo>[^/@]+?)(?:\.git)?(?:@(?P<branch>[^/]+))?$"
+    r"^git\+ssh://git@(?P<host>[^/]+)/(?P<org>[^/]+)/(?P<repo>[^/@]+?)(?:\.git)?(?:@(?P<ref>[^/]+))?$"
 )
+"""Git SSH URL with git user and optional ref."""
 
 
 @dataclass
 class GitUrl:
     host: str
     org: str
     repo: str
-    branch: str
+    ref: str | None = None
+
+    def to_url(self) -> str:
+        base = f"git+ssh://git@{self.host}/{self.org}/{self.repo}.git"
+        return f"{base}@{self.ref}" if self.ref else base
 
 
 async def get_git_repository_sha(repo_url: str, env: SshEnv) -> str:
@@ -38,8 +50,8 @@ async def get_git_repository_sha(repo_url: str, env: SshEnv) -> str:
             raise RuntimeError(f"Failed to get repository SHA: {error_message}")
 
         # The output format is: "<SHA>\tHEAD"
-        sha = stdout.decode().split()[0]
-        return sha
+        ref = stdout.decode().split()[0]
+        return ref
 
     except Exception as e:
         logger.error("Error getting repository SHA", error=str(e))
@@ -63,16 +75,73 @@ def parse_git_url(url: str, *, allowed_domains: set[str] | None = None) -> GitUr
 
     if match := GIT_SSH_URL_REGEX.match(url):
         host = match.group("host")
+        org = match.group("org")
+        repo = match.group("repo")
+        ref = match.group("ref")
+
+        if (
+            not isinstance(host, str)
+            or not isinstance(org, str)
+            or not isinstance(repo, str)
+        ):
+            raise ValueError(f"Invalid Git URL: {url}")
+
         if allowed_domains and host not in allowed_domains:
             raise ValueError(
                 f"Domain {host} not in allowed domains. Must be configured in `git_allowed_domains` organization setting."
             )
 
-        return GitUrl(
-            host=host,
-            org=match.group("org"),
-            repo=match.group("repo"),
-            branch=match.group("branch") or "main",
-        )
+        return GitUrl(host=host, org=org, repo=repo, ref=ref)
 
     raise ValueError(f"Unsupported URL format: {url}. Must be a valid Git SSH URL.")
+
+
+async def prepare_git_url(role: Role | None = None) -> GitUrl | None:
+    """Construct the runtime environment
+    Deps:
+    In the new pull-model registry, the execution environment is ALL the registries
+    1. Tracecat registry
+    2. User's custom template registry
+    3. User's custom UDF registry (github)
+
+    Why?
+    Since we no longer depend on the user to push to executor, the db repos are now
+    the source of truth.
+    """
+    role = role or ctx_role.get()
+
+    # Handle the git repo
+    url = await get_setting(
+        "git_repo_url",
+        # TODO: Deprecate in future version
+        default=config.TRACECAT__REMOTE_REPOSITORY_URL,
+    )
+    if not url or not isinstance(url, str):
+        logger.debug("No git URL found")
+        return None
+
+    logger.debug("Runtime environment", url=url)
+
+    allowed_domains_setting = await get_setting(
+        "git_allowed_domains",
+        # TODO: Deprecate in future version
+        default=config.TRACECAT__ALLOWED_GIT_DOMAINS,
+    )
+    allowed_domains = cast(set[str], allowed_domains_setting or {"github.com"})
+
+    # Grab the sha
+    # Find the repository that has the same origin
+    sha = None
+    async with RegistryReposService.with_session(role=role) as service:
+        repo = await service.get_repository(origin=url)
+        sha = repo.commit_sha if repo else None
+
+    try:
+        # Validate
+        git_url = parse_git_url(url, allowed_domains=allowed_domains)
+    except ValueError as e:
+        raise TracecatSettingsError(
+            "Invalid Git repository URL. Please provide a valid Git SSH URL (git+ssh)."
+        ) from e
+    git_url.ref = sha
+    return git_url

tracecat/identifiers/__init__.py

@@ -73,6 +73,7 @@
     "tracecat-schedule-runner",
     "tracecat-service",
     "tracecat-executor",
+    "tracecat-bootstrap",
 ]
 
 __all__ = [

tracecat/registry/repository.py

@@ -263,7 +263,6 @@ async def load_from_origin(self, commit_sha: str | None = None) -> str | None:
             host = git_url.host
             org = git_url.org
             repo_name = git_url.repo
-            branch = git_url.branch
         except ValueError as e:
             raise RegistryError(
                 "Invalid Git repository URL. Please provide a valid Git SSH URL (git+ssh)."
@@ -283,7 +282,6 @@ async def load_from_origin(self, commit_sha: str | None = None) -> str | None:
             org=org,
             repo=repo_name,
             package_name=package_name,
-            ref=branch,
         )
 
         cleaned_url = self.safe_remote_url(self._origin)

tracecat/settings/service.py

@@ -10,6 +10,7 @@
 
 from tracecat import config
 from tracecat.authz.controls import require_access_level
+from tracecat.contexts import ctx_role
 from tracecat.db.schemas import OrganizationSetting
 from tracecat.logger import logger
 from tracecat.secrets.encryption import decrypt_value, encrypt_value
@@ -264,6 +265,7 @@ async def get_setting(
     default: Any | None = None,
 ) -> Any | None:
     """Shorthand to get a setting value from the database."""
+    role = role or ctx_role.get()
 
     # If we have an environment override, use it
     if override_val := get_setting_override(key):