feat(engine+infra): Executor service

.env.example

@@ -7,7 +7,7 @@ PUBLIC_APP_URL=http://localhost
 PUBLIC_API_URL=http://localhost/api
 SAML_SP_ACS_URL=${PUBLIC_API_URL}/auth/saml/acs
 INTERNAL_API_URL=http://api:8000
-
+INTERNAL_EXECUTOR_URL=http://executor:8000
 # -- Caddy env vars ---
 BASE_DOMAIN=:80
 # Note: replace with your server's IP address
@@ -29,18 +29,17 @@ TRACECAT__SIGNING_SECRET=your-tracecat-signing-secret
 TRACECAT__API_URL=${INTERNAL_API_URL}
 # Root path to deal with extra path prefix behind the reverse proxy
 TRACECAT__API_ROOT_PATH=/api
-# Public Runner URL
+# This the public URL for the frontend
+TRACECAT__PUBLIC_APP_URL=${PUBLIC_APP_URL}
 # This is the public URL for incoming webhooks
 # If you wish to expose your webhooks to the internet, you can use a tunneling service like ngrok.
 # If using ngrok, run `ngrok http --domain=INSERT_STATIC_NGROK_DOMAIN_HERE 8001`
 # to start ngrok and update this with the forwarding URL
-TRACECAT__PUBLIC_RUNNER_URL=${PUBLIC_API_URL}
+TRACECAT__PUBLIC_API_URL=${PUBLIC_API_URL}
 # CORS (comman separated string of allowed origins)
 TRACECAT__ALLOW_ORIGINS=http://localhost:3000,${PUBLIC_APP_URL}
 # Postgres SSL model
 TRACECAT__DB_SSLMODE=disable
-TRACECAT__PUBLIC_APP_URL=${PUBLIC_APP_URL}
-TRACECAT__PUBLIC_API_URL=${PUBLIC_API_URL}
 
 # --- Postgres ---
 TRACECAT__POSTGRES_USER=postgres

.github/workflows/build-push-images.yml

@@ -12,7 +12,14 @@ permissions:
 
 jobs:
   push-api-to-ghcr:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-arm64-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -48,7 +55,7 @@ jobs:
         with:
           context: .
           push: true
-          platforms: linux/amd64,linux/arm64
+          platforms: ${{ matrix.platform }}
           tags: |
             ${{ steps.meta.outputs.tags }}
             ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && 'ghcr.io/tracecathq/tracecat:latest' || '' }}
@@ -57,7 +64,14 @@ jobs:
           cache-to: type=gha,mode=max
 
   push-ui-to-ghcr:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-arm64-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -106,7 +120,7 @@ jobs:
             NEXT_SERVER_API_URL=${{ env.NEXT_SERVER_API_URL }}
             NODE_ENV=${{ env.NODE_ENV }}
           push: true
-          platforms: linux/amd64,linux/arm64
+          platforms: ${{ matrix.platform }}
           tags: ${{ steps.meta.outputs.tags }}
             ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && 'ghcr.io/tracecathq/tracecat-ui:latest' || '' }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test-python.yml

@@ -10,7 +10,7 @@ on:
       - pyproject.toml
       - .github/workflows/test-python.yml
   pull_request:
-    branches: ["main"]
+    branches: ["main", "staging"]
     paths:
       - tracecat/**
       - registry/**
@@ -21,8 +21,7 @@ on:
     inputs:
       git-ref:
         description: "Git Ref (Optional)"
-        required: false
-        default: "main"
+        required: true
 
 permissions:
   contents: read
@@ -126,7 +125,7 @@ jobs:
       - name: Start Docker services
         env:
           TRACECAT__UNSAFE_DISABLE_SM_MASKING: "true"
-        run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker postgres_db caddy
+        run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker executor postgres_db caddy
 
       - name: Install dependencies
         run: |

Caddyfile

@@ -1,5 +1,8 @@
 {$BASE_DOMAIN} {
 	bind {$ADDRESS} # Binds to all available network interfaces if not specified
+	handle_path /api/executor* {
+		reverse_proxy http://executor:8000
+	}
 	handle_path /api* {
 		reverse_proxy http://api:8000
 	}

deployments/aws/ecs/ecs-api.tf

@@ -28,31 +28,14 @@ resource "aws_ecs_task_definition" "api_task_definition" {
           awslogs-stream-prefix = "api"
         }
       }
-      environment = concat(local.api_env, [
-        {
-          name  = "TRACECAT__DB_ENDPOINT"
-          value = local.core_db_hostname
-        },
-        {
-          name  = "TRACECAT__REMOTE_REPOSITORY_PACKAGE_NAME"
-          value = var.remote_repository_package_name
-        },
-        {
-          name  = "TRACECAT__REMOTE_REPOSITORY_URL"
-          value = var.remote_repository_url
-        }
-      ])
-      secrets = local.tracecat_secrets
+      environment = local.api_env
+      secrets     = local.tracecat_api_secrets
       dockerPullConfig = {
         maxAttempts = 3
         backoffTime = 10
       }
     }
   ])
-
-  depends_on = [
-    aws_ecs_service.temporal_service
-  ]
 }
 
 resource "aws_ecs_service" "tracecat_api" {
@@ -67,6 +50,7 @@ resource "aws_ecs_service" "tracecat_api" {
     subnets = var.private_subnet_ids
     security_groups = [
       aws_security_group.core.id,
+      aws_security_group.caddy.id,
       aws_security_group.core_db.id,
     ]
   }
@@ -98,6 +82,7 @@ resource "aws_ecs_service" "tracecat_api" {
 
   depends_on = [
     aws_ecs_service.temporal_service,
+    aws_ecs_service.tracecat_executor
   ]
 
 }

deployments/aws/ecs/ecs-caddy.tf

@@ -62,8 +62,7 @@ resource "aws_ecs_service" "tracecat_caddy" {
   network_configuration {
     subnets = var.private_subnet_ids
     security_groups = [
-      aws_security_group.caddy.id,
-      aws_security_group.core.id
+      aws_security_group.caddy.id
     ]
   }
 

deployments/aws/ecs/ecs-executor.tf

@@ -0,0 +1,91 @@
+# ECS Task Definition for Executor Service
+resource "aws_ecs_task_definition" "executor_task_definition" {
+  family                   = "TracecatExecutorTaskDefinition"
+  network_mode             = "awsvpc"
+  requires_compatibilities = ["FARGATE"]
+  cpu                      = var.executor_cpu
+  memory                   = var.executor_memory
+  execution_role_arn       = aws_iam_role.worker_execution.arn
+  task_role_arn            = aws_iam_role.api_worker_task.arn
+
+  container_definitions = jsonencode([
+    {
+      name  = "TracecatExecutorContainer"
+      image = "${var.tracecat_image}:${local.tracecat_image_tag}"
+      command = [
+        "python",
+        "-m",
+        "uvicorn",
+        "tracecat.api.executor:app",
+        "--host",
+        "0.0.0.0",
+        "--port",
+        "8002"
+      ]
+      portMappings = [
+        {
+          containerPort = 8002
+          hostPort      = 8002
+          name          = "executor"
+          appProtocol   = "http"
+        }
+      ]
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          awslogs-group         = aws_cloudwatch_log_group.tracecat_log_group.name
+          awslogs-region        = var.aws_region
+          awslogs-stream-prefix = "executor"
+        }
+      }
+      environment = local.executor_env
+      secrets     = local.tracecat_base_secrets
+      dockerPullConfig = {
+        maxAttempts = 3
+        backoffTime = 10
+      }
+    }
+  ])
+}
+
+resource "aws_ecs_service" "tracecat_executor" {
+  name                 = "tracecat-executor"
+  cluster              = aws_ecs_cluster.tracecat_cluster.id
+  task_definition      = aws_ecs_task_definition.executor_task_definition.arn
+  launch_type          = "FARGATE"
+  desired_count        = 1
+  force_new_deployment = var.force_new_deployment
+
+  network_configuration {
+    subnets = var.private_subnet_ids
+    security_groups = [
+      aws_security_group.core.id,
+      aws_security_group.core_db.id,
+    ]
+  }
+
+  service_connect_configuration {
+    enabled   = true
+    namespace = local.local_dns_namespace
+    service {
+      port_name      = "executor"
+      discovery_name = "executor-service"
+      timeout {
+        per_request_timeout_seconds = 120
+      }
+      client_alias {
+        port     = 8002
+        dns_name = "executor-service"
+      }
+    }
+
+    log_configuration {
+      log_driver = "awslogs"
+      options = {
+        awslogs-group         = aws_cloudwatch_log_group.tracecat_log_group.name
+        awslogs-region        = var.aws_region
+        awslogs-stream-prefix = "service-connect-executor"
+      }
+    }
+  }
+}

deployments/aws/ecs/ecs-ui.tf

@@ -48,6 +48,7 @@ resource "aws_ecs_service" "tracecat_ui" {
     subnets = var.private_subnet_ids
     security_groups = [
       aws_security_group.core.id,
+      aws_security_group.caddy.id
     ]
   }
 

deployments/aws/ecs/ecs-worker.tf

@@ -29,24 +29,14 @@ resource "aws_ecs_task_definition" "worker_task_definition" {
           awslogs-stream-prefix = "worker"
         }
       }
-      environment = concat(local.worker_env, [
-        {
-          name  = "TRACECAT__DB_ENDPOINT"
-          value = local.core_db_hostname
-        }
-      ])
-      secrets = local.tracecat_secrets
+      environment = local.worker_env
+      secrets     = local.tracecat_base_secrets
       dockerPullConfig = {
         maxAttempts = 3
         backoffTime = 30
       }
     }
   ])
-
-  depends_on = [
-    aws_ecs_service.temporal_service,
-    aws_ecs_task_definition.temporal_task_definition,
-  ]
 }
 
 resource "aws_ecs_service" "tracecat_worker" {
@@ -92,6 +82,6 @@ resource "aws_ecs_service" "tracecat_worker" {
 
   depends_on = [
     aws_ecs_service.temporal_service,
-    aws_ecs_service.tracecat_api,
+    aws_ecs_service.tracecat_executor
   ]
 }

deployments/aws/ecs/iam.tf

@@ -49,7 +49,11 @@ resource "aws_iam_policy" "secrets_access" {
           var.tracecat_service_key_arn,
           var.tracecat_signing_secret_arn,
           var.oauth_client_id_arn,
-          var.oauth_client_secret_arn
+          var.oauth_client_secret_arn,
+          var.saml_idp_entity_id_arn,
+          var.saml_idp_redirect_url_arn,
+          var.saml_idp_certificate_arn,
+          var.saml_idp_metadata_url_arn,
         ])
       }
     ]
@@ -110,6 +114,12 @@ resource "aws_iam_role_policy_attachment" "worker_execution_secrets" {
   role       = aws_iam_role.worker_execution.name
 }
 
+# Executor execution role
+resource "aws_iam_role" "executor_execution" {
+  name               = "TracecatExecutorExecutionRole"
+  assume_role_policy = data.aws_iam_policy_document.assume_role.json
+}
+
 # UI execution role
 resource "aws_iam_role" "ui_execution" {
   name               = "TracecatUIExecutionRole"