refactor(engine): Move registry management endpoints into api service…

… + restructure registry as executor service (#590)

.env.example

@@ -7,7 +7,7 @@ PUBLIC_APP_URL=http://localhost
 PUBLIC_API_URL=http://localhost/api
 SAML_SP_ACS_URL=${PUBLIC_API_URL}/auth/saml/acs
 INTERNAL_API_URL=http://api:8000
-INTERNAL_REGISTRY_URL=http://registry:8000
+INTERNAL_EXECUTOR_URL=http://executor:8000
 # -- Caddy env vars ---
 BASE_DOMAIN=:80
 # Note: replace with your server's IP address
@@ -29,18 +29,17 @@ TRACECAT__SIGNING_SECRET=your-tracecat-signing-secret
 TRACECAT__API_URL=${INTERNAL_API_URL}
 # Root path to deal with extra path prefix behind the reverse proxy
 TRACECAT__API_ROOT_PATH=/api
-# Public Runner URL
+# This the public URL for the frontend
+TRACECAT__PUBLIC_APP_URL=${PUBLIC_APP_URL}
 # This is the public URL for incoming webhooks
 # If you wish to expose your webhooks to the internet, you can use a tunneling service like ngrok.
 # If using ngrok, run `ngrok http --domain=INSERT_STATIC_NGROK_DOMAIN_HERE 8001`
 # to start ngrok and update this with the forwarding URL
-TRACECAT__PUBLIC_RUNNER_URL=${PUBLIC_API_URL}
+TRACECAT__PUBLIC_API_URL=${PUBLIC_API_URL}
 # CORS (comman separated string of allowed origins)
 TRACECAT__ALLOW_ORIGINS=http://localhost:3000,${PUBLIC_APP_URL}
 # Postgres SSL model
 TRACECAT__DB_SSLMODE=disable
-TRACECAT__PUBLIC_APP_URL=${PUBLIC_APP_URL}
-TRACECAT__PUBLIC_API_URL=${PUBLIC_API_URL}
 
 # --- Postgres ---
 TRACECAT__POSTGRES_USER=postgres

.github/workflows/test-python.yml

@@ -10,7 +10,7 @@ on:
       - pyproject.toml
       - .github/workflows/test-python.yml
   pull_request:
-    branches: ["main"]
+    branches: ["main", "staging"]
     paths:
       - tracecat/**
       - registry/**
@@ -21,8 +21,7 @@ on:
     inputs:
       git-ref:
         description: "Git Ref (Optional)"
-        required: false
-        default: "main"
+        required: true
 
 permissions:
   contents: read
@@ -126,7 +125,7 @@ jobs:
       - name: Start Docker services
         env:
           TRACECAT__UNSAFE_DISABLE_SM_MASKING: "true"
-        run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker registry postgres_db caddy
+        run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker executor postgres_db caddy
 
       - name: Install dependencies
         run: |

Caddyfile

@@ -1,7 +1,7 @@
 {$BASE_DOMAIN} {
 	bind {$ADDRESS} # Binds to all available network interfaces if not specified
-	handle_path /api/registry* {
-		reverse_proxy http://registry:8000
+	handle_path /api/executor* {
+		reverse_proxy http://executor:8000
 	}
 	handle_path /api* {
 		reverse_proxy http://api:8000

deployments/aws/ecs/ecs-api.tf

@@ -50,6 +50,7 @@ resource "aws_ecs_service" "tracecat_api" {
     subnets = var.private_subnet_ids
     security_groups = [
       aws_security_group.core.id,
+      aws_security_group.caddy.id,
       aws_security_group.core_db.id,
     ]
   }
@@ -81,7 +82,7 @@ resource "aws_ecs_service" "tracecat_api" {
 
   depends_on = [
     aws_ecs_service.temporal_service,
-    aws_ecs_service.tracecat_registry
+    aws_ecs_service.tracecat_executor
   ]
 
 }

deployments/aws/ecs/ecs-caddy.tf

@@ -62,8 +62,7 @@ resource "aws_ecs_service" "tracecat_caddy" {
   network_configuration {
     subnets = var.private_subnet_ids
     security_groups = [
-      aws_security_group.caddy.id,
-      aws_security_group.core.id
+      aws_security_group.caddy.id
     ]
   }
 

deployments/aws/ecs/ecs-registry.tf → deployments/aws/ecs/ecs-executor.tf

@@ -1,32 +1,32 @@
-# ECS Task Definition for Registry Service
-resource "aws_ecs_task_definition" "registry_task_definition" {
-  family                   = "TracecatRegistryTaskDefinition"
+# ECS Task Definition for Executor Service
+resource "aws_ecs_task_definition" "executor_task_definition" {
+  family                   = "TracecatExecutorTaskDefinition"
   network_mode             = "awsvpc"
   requires_compatibilities = ["FARGATE"]
-  cpu                      = var.registry_cpu
-  memory                   = var.registry_memory
+  cpu                      = var.executor_cpu
+  memory                   = var.executor_memory
   execution_role_arn       = aws_iam_role.worker_execution.arn
   task_role_arn            = aws_iam_role.api_worker_task.arn
 
   container_definitions = jsonencode([
     {
-      name  = "TracecatRegistryContainer"
+      name  = "TracecatExecutorContainer"
       image = "${var.tracecat_image}:${local.tracecat_image_tag}"
       command = [
         "python",
         "-m",
         "uvicorn",
-        "tracecat.api.registry:app",
+        "tracecat.api.executor:app",
         "--host",
         "0.0.0.0",
         "--port",
-        "8002"
+        "8000"
       ]
       portMappings = [
         {
           containerPort = 8002
           hostPort      = 8002
-          name          = "registry"
+          name          = "executor"
           appProtocol   = "http"
         }
       ]
@@ -35,10 +35,10 @@ resource "aws_ecs_task_definition" "registry_task_definition" {
         options = {
           awslogs-group         = aws_cloudwatch_log_group.tracecat_log_group.name
           awslogs-region        = var.aws_region
-          awslogs-stream-prefix = "registry"
+          awslogs-stream-prefix = "executor"
         }
       }
-      environment = local.registry_env
+      environment = local.executor_env
       secrets     = local.tracecat_base_secrets
       dockerPullConfig = {
         maxAttempts = 3
@@ -48,10 +48,10 @@ resource "aws_ecs_task_definition" "registry_task_definition" {
   ])
 }
 
-resource "aws_ecs_service" "tracecat_registry" {
-  name                 = "tracecat-registry"
+resource "aws_ecs_service" "tracecat_executor" {
+  name                 = "tracecat-executor"
   cluster              = aws_ecs_cluster.tracecat_cluster.id
-  task_definition      = aws_ecs_task_definition.registry_task_definition.arn
+  task_definition      = aws_ecs_task_definition.executor_task_definition.arn
   launch_type          = "FARGATE"
   desired_count        = 1
   force_new_deployment = var.force_new_deployment
@@ -68,14 +68,14 @@ resource "aws_ecs_service" "tracecat_registry" {
     enabled   = true
     namespace = local.local_dns_namespace
     service {
-      port_name      = "registry"
-      discovery_name = "registry-service"
+      port_name      = "executor"
+      discovery_name = "executor-service"
       timeout {
         per_request_timeout_seconds = 120
       }
       client_alias {
         port     = 8002
-        dns_name = "registry-service"
+        dns_name = "executor-service"
       }
     }
 
@@ -84,7 +84,7 @@ resource "aws_ecs_service" "tracecat_registry" {
       options = {
         awslogs-group         = aws_cloudwatch_log_group.tracecat_log_group.name
         awslogs-region        = var.aws_region
-        awslogs-stream-prefix = "service-connect-registry"
+        awslogs-stream-prefix = "service-connect-executor"
       }
     }
   }

deployments/aws/ecs/ecs-ui.tf

@@ -48,6 +48,7 @@ resource "aws_ecs_service" "tracecat_ui" {
     subnets = var.private_subnet_ids
     security_groups = [
       aws_security_group.core.id,
+      aws_security_group.caddy.id
     ]
   }
 

deployments/aws/ecs/ecs-worker.tf

@@ -81,6 +81,7 @@ resource "aws_ecs_service" "tracecat_worker" {
   }
 
   depends_on = [
-    aws_ecs_service.temporal_service
+    aws_ecs_service.temporal_service,
+    aws_ecs_service.tracecat_executor
   ]
 }

deployments/aws/ecs/iam.tf

@@ -114,9 +114,9 @@ resource "aws_iam_role_policy_attachment" "worker_execution_secrets" {
   role       = aws_iam_role.worker_execution.name
 }
 
-# Registry execution role
-resource "aws_iam_role" "registry_execution" {
-  name               = "TracecatRegistryExecutionRole"
+# Executor execution role
+resource "aws_iam_role" "executor_execution" {
+  name               = "TracecatExecutorExecutionRole"
   assume_role_policy = data.aws_iam_policy_document.assume_role.json
 }
 

deployments/aws/ecs/locals.tf

@@ -11,7 +11,7 @@ locals {
   public_api_url         = "https://${var.domain_name}/api"
   saml_acs_url           = "https://${var.domain_name}/api/auth/saml/acs"
   internal_api_url       = "http://api-service:8000"      # Service connect DNS name
-  internal_registry_url  = "http://registry-service:8002" # Service connect DNS name
+  internal_executor_url  = "http://executor-service:8002" # Service connect DNS name
   temporal_cluster_url   = "temporal-service:7233"
   temporal_cluster_queue = "tracecat-task-queue"
   allow_origins          = "${var.domain_name},http://ui-service:3000" # Allow api service and public app to access the API
@@ -41,10 +41,10 @@ locals {
       TRACECAT__AUTH_TYPES                     = var.auth_types
       TRACECAT__DB_ENDPOINT                    = local.core_db_hostname
       TRACECAT__PUBLIC_APP_URL                 = local.public_app_url
-      TRACECAT__PUBLIC_RUNNER_URL              = local.public_api_url
+      TRACECAT__PUBLIC_API_URL                 = local.public_api_url
       TRACECAT__REMOTE_REPOSITORY_PACKAGE_NAME = var.remote_repository_package_name
       TRACECAT__REMOTE_REPOSITORY_URL          = var.remote_repository_url
-      TRACECAT__REGISTRY_URL                   = local.internal_registry_url
+      TRACECAT__EXECUTOR_URL                   = local.internal_executor_url
     }, local.tracecat_db_configs) :
     { name = k, value = tostring(v) }
   ]
@@ -56,16 +56,16 @@ locals {
       TRACECAT__API_ROOT_PATH      = "/api"
       TRACECAT__APP_ENV            = var.tracecat_app_env
       TRACECAT__DB_ENDPOINT        = local.core_db_hostname
-      TRACECAT__PUBLIC_RUNNER_URL  = local.public_api_url
+      TRACECAT__PUBLIC_API_URL     = local.public_api_url
       TEMPORAL__CLUSTER_URL        = local.temporal_cluster_url
       TEMPORAL__CLUSTER_QUEUE      = local.temporal_cluster_queue
       TEMPORAL__CLIENT_RPC_TIMEOUT = var.temporal_client_rpc_timeout
-      TRACECAT__REGISTRY_URL       = local.internal_registry_url
+      TRACECAT__EXECUTOR_URL       = local.internal_executor_url
     }, local.tracecat_db_configs) :
     { name = k, value = tostring(v) }
   ]
 
-  registry_env = [
+  executor_env = [
     for k, v in merge({
       LOG_LEVEL                                = var.log_level
       TRACECAT__APP_ENV                        = var.tracecat_app_env

deployments/aws/ecs/security_groups.tf

@@ -31,12 +31,29 @@ resource "aws_security_group" "caddy" {
   vpc_id      = var.vpc_id
 
   ingress {
+    description     = "Allow inbound access from ALB to port 80 (Caddy) only"
     protocol        = "tcp"
     from_port       = 80
     to_port         = 80
     security_groups = [aws_security_group.alb.id]
   }
 
+  ingress {
+    description = "Allow Caddy to forward traffic to API service only"
+    protocol    = "tcp"
+    from_port   = 8000
+    to_port     = 8000
+    self        = true
+  }
+
+  ingress {
+    description = "Allow Caddy to forward traffic to UI service only"
+    protocol    = "tcp"
+    from_port   = 3000
+    to_port     = 3000
+    self        = true
+  }
+
   egress {
     protocol    = "-1"
     from_port   = 0
@@ -67,7 +84,7 @@ resource "aws_security_group" "core" {
   }
 
   ingress {
-    description = "Allow internal traffic to the Tracecat Registry service on port 8002"
+    description = "Allow internal traffic to the Tracecat Executor service on port 8000"
     from_port   = 8002
     to_port     = 8002
     protocol    = "tcp"

deployments/aws/ecs/variables.tf

@@ -212,12 +212,12 @@ variable "worker_memory" {
   default = "512"
 }
 
-variable "registry_cpu" {
+variable "executor_cpu" {
   type    = string
   default = "256"
 }
 
-variable "registry_memory" {
+variable "executor_memory" {
   type    = string
   default = "512"
 }

deployments/aws/main.tf

@@ -78,8 +78,8 @@ module "ecs" {
   api_memory                  = var.api_memory
   worker_cpu                  = var.worker_cpu
   worker_memory               = var.worker_memory
-  registry_cpu                = var.registry_cpu
-  registry_memory             = var.registry_memory
+  executor_cpu                = var.executor_cpu
+  executor_memory             = var.executor_memory
   ui_cpu                      = var.ui_cpu
   ui_memory                   = var.ui_memory
   temporal_cpu                = var.temporal_cpu

deployments/aws/variables.tf

@@ -145,12 +145,12 @@ variable "worker_memory" {
   default = "512"
 }
 
-variable "registry_cpu" {
+variable "executor_cpu" {
   type    = string
   default = "256"
 }
 
-variable "registry_memory" {
+variable "executor_memory" {
   type    = string
   default = "512"
 }