refactor(engine): Move registry management endpoints into api service…

… + restructure registry as executor service (#590)
TracecatHQ · Dec 7, 2024 · 6d84a98 · 6d84a98
1 parent 2ae505c
commit 6d84a98
Show file tree

Hide file tree

Showing 21 changed files with 500 additions and 675 deletions.
diff --git a/.env.example b/.env.example
@@ -7,7 +7,7 @@ PUBLIC_APP_URL=http://localhost
 PUBLIC_API_URL=http://localhost/api
 SAML_SP_ACS_URL=${PUBLIC_API_URL}/auth/saml/acs
 INTERNAL_API_URL=http://api:8000
-INTERNAL_REGISTRY_URL=http://registry:8000
+INTERNAL_EXECUTOR_URL=http://executor:8000
 # -- Caddy env vars ---
 BASE_DOMAIN=:80
 # Note: replace with your server's IP address

diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -10,7 +10,7 @@ on:
       - pyproject.toml
       - .github/workflows/test-python.yml
   pull_request:
-    branches: ["main"]
+    branches: ["main", "staging"]
     paths:
       - tracecat/**
       - registry/**
@@ -21,8 +21,7 @@ on:
     inputs:
       git-ref:
         description: "Git Ref (Optional)"
-        required: false
-        default: "main"
+        required: true
 
 permissions:
   contents: read
@@ -126,7 +125,7 @@ jobs:
       - name: Start Docker services
         env:
           TRACECAT__UNSAFE_DISABLE_SM_MASKING: "true"
-        run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker registry postgres_db caddy
+        run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker executor postgres_db caddy
 
       - name: Install dependencies
         run: |

diff --git a/Caddyfile b/Caddyfile
@@ -1,7 +1,7 @@
 {$BASE_DOMAIN} {
 	bind {$ADDRESS} # Binds to all available network interfaces if not specified
-	handle_path /api/registry* {
-		reverse_proxy http://registry:8000
+	handle_path /api/executor* {
+		reverse_proxy http://executor:8000
 	}
 	handle_path /api* {
 		reverse_proxy http://api:8000

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -36,7 +36,7 @@ services:
       TRACECAT__AUTH_TYPES: ${TRACECAT__AUTH_TYPES}
       TRACECAT__AUTH_ALLOWED_DOMAINS: ${TRACECAT__AUTH_ALLOWED_DOMAINS}
       TRACECAT__AUTH_MIN_PASSWORD_LENGTH: ${TRACECAT__AUTH_MIN_PASSWORD_LENGTH}
-      TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
+      TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
       OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID}
       OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET}
       USER_AUTH_SECRET: ${USER_AUTH_SECRET}
@@ -63,6 +63,7 @@ services:
       - ./alembic:/app/alembic
     depends_on:
       - ollama
+      - executor
 
   worker:
     build:
@@ -78,7 +79,7 @@ services:
       TRACECAT__DB_SSLMODE: ${TRACECAT__DB_SSLMODE}
       TRACECAT__DB_URI: ${TRACECAT__DB_URI} # Sensitive
       TRACECAT__PUBLIC_RUNNER_URL: ${TRACECAT__PUBLIC_RUNNER_URL}
-      TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
+      TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
       TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} # Sensitive
       TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} # Sensitive
       # Temporal
@@ -87,9 +88,9 @@ services:
     volumes:
       - ./tracecat:/app/tracecat
       - ./registry:/app/registry
-    entrypoint: ["python", "tracecat/dsl/worker.py"]
+    command: ["python", "tracecat/dsl/worker.py"]
 
-  registry:
+  executor:
     build:
       context: .
       dockerfile: Dockerfile.dev
@@ -112,13 +113,12 @@ services:
       OLLAMA__API_URL: ${OLLAMA__API_URL}
     volumes:
       - ./tracecat:/app/tracecat
-      - ./registry:/app/registry
-    entrypoint:
+    command:
       [
         "python",
         "-m",
         "uvicorn",
-        "tracecat.api.registry:app",
+        "tracecat.api.executor:app",
         "--host",
         "0.0.0.0",
         "--port",

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -38,7 +38,7 @@ services:
       TRACECAT__AUTH_TYPES: ${TRACECAT__AUTH_TYPES}
       TRACECAT__AUTH_ALLOWED_DOMAINS: ${TRACECAT__AUTH_ALLOWED_DOMAINS}
       TRACECAT__AUTH_MIN_PASSWORD_LENGTH: ${TRACECAT__AUTH_MIN_PASSWORD_LENGTH}
-      TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
+      TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
       OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID}
       OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET}
       USER_AUTH_SECRET: ${USER_AUTH_SECRET}
@@ -75,15 +75,15 @@ services:
       TRACECAT__DB_SSLMODE: ${TRACECAT__DB_SSLMODE}
       TRACECAT__DB_URI: ${TRACECAT__DB_URI} # Sensitive
       TRACECAT__PUBLIC_RUNNER_URL: ${TRACECAT__PUBLIC_RUNNER_URL}
-      TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL}
+      TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL}
       TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} # Sensitive
       TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} # Sensitive
       # Temporal
       TEMPORAL__CLUSTER_URL: ${TEMPORAL__CLUSTER_URL}
       TEMPORAL__CLUSTER_QUEUE: ${TEMPORAL__CLUSTER_QUEUE}
     command: ["python", "tracecat/dsl/worker.py"]
 
-  registry:
+  executor:
     image: ghcr.io/tracecathq/tracecat:${TRACECAT__IMAGE_TAG:-0.16.0}
     restart: unless-stopped
     networks:
@@ -108,7 +108,7 @@ services:
         "python",
         "-m",
         "uvicorn",
-        "tracecat.api.registry:app",
+        "tracecat.api.executor:app",
         "--host",
         "0.0.0.0",
         "--port",

diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts
@@ -173,110 +173,7 @@ export const $ActionRetryPolicy = {
     title: 'ActionRetryPolicy'
 } as const;
 
-export const $ActionStatement_Input = {
-    properties: {
-        id: {
-            anyOf: [
-                {
-                    type: 'string'
-                },
-                {
-                    type: 'null'
-                }
-            ],
-            title: 'Id',
-            description: 'The action ID. If this is populated means there is a corresponding actionin the database `Action` table.'
-        },
-        ref: {
-            type: 'string',
-            pattern: '^[a-z0-9_]+$',
-            title: 'Ref',
-            description: 'Unique reference for the task'
-        },
-        description: {
-            type: 'string',
-            title: 'Description',
-            default: ''
-        },
-        action: {
-            type: 'string',
-            pattern: '^[a-z0-9_.]+$',
-            title: 'Action',
-            description: 'Action type. Equivalent to the UDF key.'
-        },
-        args: {
-            type: 'object',
-            title: 'Args',
-            description: 'Arguments for the action'
-        },
-        depends_on: {
-            items: {
-                type: 'string'
-            },
-            type: 'array',
-            title: 'Depends On',
-            description: 'Task dependencies'
-        },
-        run_if: {
-            anyOf: [
-                {
-                    type: 'string'
-                },
-                {
-                    type: 'null'
-                }
-            ],
-            title: 'Run If',
-            description: 'Condition to run the task'
-        },
-        for_each: {
-            anyOf: [
-                {
-                    type: 'string'
-                },
-                {
-                    items: {
-                        type: 'string'
-                    },
-                    type: 'array'
-                },
-                {
-                    type: 'null'
-                }
-            ],
-            title: 'For Each',
-            description: 'Iterate over a list of items and run the task for each item.'
-        },
-        retry_policy: {
-            allOf: [
-                {
-                    '$ref': '#/components/schemas/ActionRetryPolicy'
-                }
-            ],
-            description: 'Retry policy for the action.'
-        },
-        start_delay: {
-            type: 'number',
-            title: 'Start Delay',
-            description: 'Delay before starting the action in seconds.',
-            default: 0
-        },
-        join_strategy: {
-            allOf: [
-                {
-                    '$ref': '#/components/schemas/JoinStrategy'
-                }
-            ],
-            description: 'The strategy to use when joining on this task. By default, all branches must complete successfully before the join task can complete.',
-            default: 'all'
-        }
-    },
-    type: 'object',
-    required: ['ref', 'action'],
-    title: 'ActionStatement'
-} as const;
-
-export const $ActionStatement_Output = {
+export const $ActionStatement = {
     properties: {
         ref: {
             type: 'string',
@@ -758,10 +655,15 @@ export const $DSLContext = {
         },
         ENV: {
             '$ref': '#/components/schemas/DSLEnvironment'
+        },
+        SECRETS: {
+            type: 'object',
+            title: 'Secrets'
         }
     },
     type: 'object',
-    title: 'DSLContext'
+    title: 'DSLContext',
+    description: 'DSL Context. Contains all the context needed to execute a DSL workflow.'
 } as const;
 
 export const $DSLEntrypoint = {
@@ -837,7 +739,7 @@ export const $DSLInput = {
         },
         actions: {
             items: {
-                '$ref': '#/components/schemas/ActionStatement-Output'
+                '$ref': '#/components/schemas/ActionStatement'
             },
             type: 'array',
             title: 'Actions'
@@ -1118,7 +1020,7 @@ export const $EventGroup = {
         action_input: {
             anyOf: [
                 {
-                    '$ref': '#/components/schemas/RunActionInput-Output'
+                    '$ref': '#/components/schemas/RunActionInput'
                 },
                 {
                     '$ref': '#/components/schemas/DSLRunArgs'
@@ -1333,7 +1235,7 @@ export const $GetWorkflowDefinitionActivityInputs = {
         task: {
             anyOf: [
                 {
-                    '$ref': '#/components/schemas/ActionStatement-Output'
+                    '$ref': '#/components/schemas/ActionStatement'
                 },
                 {
                     type: 'null'
@@ -1868,18 +1770,6 @@ export const $RegistryActionUpdate = {
     description: 'API update model for a registered action.'
 } as const;
 
-export const $RegistryActionValidate = {
-    properties: {
-        args: {
-            type: 'object',
-            title: 'Args'
-        }
-    },
-    type: 'object',
-    required: ['args'],
-    title: 'RegistryActionValidate'
-} as const;
-
 export const $RegistryActionValidateResponse = {
     properties: {
         ok: {
@@ -2086,7 +1976,7 @@ export const $Role = {
         },
         service_id: {
             type: 'string',
-            enum: ['tracecat-runner', 'tracecat-api', 'tracecat-cli', 'tracecat-schedule-runner', 'tracecat-service'],
+            enum: ['tracecat-runner', 'tracecat-api', 'tracecat-cli', 'tracecat-schedule-runner', 'tracecat-service', 'tracecat-executor'],
             title: 'Service Id'
         }
     },
@@ -2119,28 +2009,10 @@ Service roles
 - A service's \`user_id\` is the user it's acting on behalf of. This can be None for internal services.`
 } as const;
 
-export const $RunActionInput_Input = {
-    properties: {
-        task: {
-            '$ref': '#/components/schemas/ActionStatement-Input'
-        },
-        exec_context: {
-            '$ref': '#/components/schemas/DSLContext'
-        },
-        run_context: {
-            '$ref': '#/components/schemas/RunContext'
-        }
-    },
-    type: 'object',
-    required: ['task', 'exec_context', 'run_context'],
-    title: 'RunActionInput',
-    description: 'This object contains all the information needed to execute an action.'
-} as const;
-
-export const $RunActionInput_Output = {
+export const $RunActionInput = {
     properties: {
         task: {
-            '$ref': '#/components/schemas/ActionStatement-Output'
+            '$ref': '#/components/schemas/ActionStatement'
         },
         exec_context: {
             '$ref': '#/components/schemas/DSLContext'