diff --git a/.env.example b/.env.example index 8f7a81f91..10d2d9f8c 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,7 @@ PUBLIC_APP_URL=http://localhost PUBLIC_API_URL=http://localhost/api SAML_SP_ACS_URL=${PUBLIC_API_URL}/auth/saml/acs INTERNAL_API_URL=http://api:8000 -INTERNAL_REGISTRY_URL=http://registry:8000 +INTERNAL_EXECUTOR_URL=http://executor:8000 # -- Caddy env vars --- BASE_DOMAIN=:80 # Note: replace with your server's IP address diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 3b29154f5..0bacf0e1e 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -10,7 +10,7 @@ on: - pyproject.toml - .github/workflows/test-python.yml pull_request: - branches: ["main"] + branches: ["main", "staging"] paths: - tracecat/** - registry/** @@ -21,8 +21,7 @@ on: inputs: git-ref: description: "Git Ref (Optional)" - required: false - default: "main" + required: true permissions: contents: read @@ -126,7 +125,7 @@ jobs: - name: Start Docker services env: TRACECAT__UNSAFE_DISABLE_SM_MASKING: "true" - run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker registry postgres_db caddy + run: docker compose -f docker-compose.dev.yml up --build --no-deps -d api worker executor postgres_db caddy - name: Install dependencies run: | diff --git a/Caddyfile b/Caddyfile index 16fe5f3bb..bf9fb62c3 100644 --- a/Caddyfile +++ b/Caddyfile @@ -1,7 +1,7 @@ {$BASE_DOMAIN} { bind {$ADDRESS} # Binds to all available network interfaces if not specified - handle_path /api/registry* { - reverse_proxy http://registry:8000 + handle_path /api/executor* { + reverse_proxy http://executor:8000 } handle_path /api* { reverse_proxy http://api:8000 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 94938730d..6852ae39c 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -36,7 +36,7 @@ services: TRACECAT__AUTH_TYPES: ${TRACECAT__AUTH_TYPES} TRACECAT__AUTH_ALLOWED_DOMAINS: ${TRACECAT__AUTH_ALLOWED_DOMAINS} TRACECAT__AUTH_MIN_PASSWORD_LENGTH: ${TRACECAT__AUTH_MIN_PASSWORD_LENGTH} - TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL} + TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL} OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID} OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET} USER_AUTH_SECRET: ${USER_AUTH_SECRET} @@ -63,6 +63,7 @@ services: - ./alembic:/app/alembic depends_on: - ollama + - executor worker: build: @@ -78,7 +79,7 @@ services: TRACECAT__DB_SSLMODE: ${TRACECAT__DB_SSLMODE} TRACECAT__DB_URI: ${TRACECAT__DB_URI} # Sensitive TRACECAT__PUBLIC_RUNNER_URL: ${TRACECAT__PUBLIC_RUNNER_URL} - TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL} + TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL} TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} # Sensitive TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} # Sensitive # Temporal @@ -87,9 +88,9 @@ services: volumes: - ./tracecat:/app/tracecat - ./registry:/app/registry - entrypoint: ["python", "tracecat/dsl/worker.py"] + command: ["python", "tracecat/dsl/worker.py"] - registry: + executor: build: context: . dockerfile: Dockerfile.dev @@ -112,13 +113,12 @@ services: OLLAMA__API_URL: ${OLLAMA__API_URL} volumes: - ./tracecat:/app/tracecat - - ./registry:/app/registry - entrypoint: + command: [ "python", "-m", "uvicorn", - "tracecat.api.registry:app", + "tracecat.api.executor:app", "--host", "0.0.0.0", "--port", diff --git a/docker-compose.yml b/docker-compose.yml index edb55d4f1..0173aaf40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,7 +38,7 @@ services: TRACECAT__AUTH_TYPES: ${TRACECAT__AUTH_TYPES} TRACECAT__AUTH_ALLOWED_DOMAINS: ${TRACECAT__AUTH_ALLOWED_DOMAINS} TRACECAT__AUTH_MIN_PASSWORD_LENGTH: ${TRACECAT__AUTH_MIN_PASSWORD_LENGTH} - TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL} + TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL} OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID} OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET} USER_AUTH_SECRET: ${USER_AUTH_SECRET} @@ -75,7 +75,7 @@ services: TRACECAT__DB_SSLMODE: ${TRACECAT__DB_SSLMODE} TRACECAT__DB_URI: ${TRACECAT__DB_URI} # Sensitive TRACECAT__PUBLIC_RUNNER_URL: ${TRACECAT__PUBLIC_RUNNER_URL} - TRACECAT__REGISTRY_URL: ${INTERNAL_REGISTRY_URL} + TRACECAT__EXECUTOR_URL: ${INTERNAL_EXECUTOR_URL} TRACECAT__SERVICE_KEY: ${TRACECAT__SERVICE_KEY} # Sensitive TRACECAT__SIGNING_SECRET: ${TRACECAT__SIGNING_SECRET} # Sensitive # Temporal @@ -83,7 +83,7 @@ services: TEMPORAL__CLUSTER_QUEUE: ${TEMPORAL__CLUSTER_QUEUE} command: ["python", "tracecat/dsl/worker.py"] - registry: + executor: image: ghcr.io/tracecathq/tracecat:${TRACECAT__IMAGE_TAG:-0.16.0} restart: unless-stopped networks: @@ -108,7 +108,7 @@ services: "python", "-m", "uvicorn", - "tracecat.api.registry:app", + "tracecat.api.executor:app", "--host", "0.0.0.0", "--port", diff --git a/frontend/src/client/schemas.gen.ts b/frontend/src/client/schemas.gen.ts index 79f56de94..820a48ae8 100644 --- a/frontend/src/client/schemas.gen.ts +++ b/frontend/src/client/schemas.gen.ts @@ -173,110 +173,7 @@ export const $ActionRetryPolicy = { title: 'ActionRetryPolicy' } as const; -export const $ActionStatement_Input = { - properties: { - id: { - anyOf: [ - { - type: 'string' - }, - { - type: 'null' - } - ], - title: 'Id', - description: 'The action ID. If this is populated means there is a corresponding actionin the database `Action` table.' - }, - ref: { - type: 'string', - pattern: '^[a-z0-9_]+$', - title: 'Ref', - description: 'Unique reference for the task' - }, - description: { - type: 'string', - title: 'Description', - default: '' - }, - action: { - type: 'string', - pattern: '^[a-z0-9_.]+$', - title: 'Action', - description: 'Action type. Equivalent to the UDF key.' - }, - args: { - type: 'object', - title: 'Args', - description: 'Arguments for the action' - }, - depends_on: { - items: { - type: 'string' - }, - type: 'array', - title: 'Depends On', - description: 'Task dependencies' - }, - run_if: { - anyOf: [ - { - type: 'string' - }, - { - type: 'null' - } - ], - title: 'Run If', - description: 'Condition to run the task' - }, - for_each: { - anyOf: [ - { - type: 'string' - }, - { - items: { - type: 'string' - }, - type: 'array' - }, - { - type: 'null' - } - ], - title: 'For Each', - description: 'Iterate over a list of items and run the task for each item.' - }, - retry_policy: { - allOf: [ - { - '$ref': '#/components/schemas/ActionRetryPolicy' - } - ], - description: 'Retry policy for the action.' - }, - start_delay: { - type: 'number', - title: 'Start Delay', - description: 'Delay before starting the action in seconds.', - default: 0 - }, - join_strategy: { - allOf: [ - { - '$ref': '#/components/schemas/JoinStrategy' - } - ], - description: 'The strategy to use when joining on this task. By default, all branches must complete successfully before the join task can complete.', - default: 'all' - } - }, - type: 'object', - required: ['ref', 'action'], - title: 'ActionStatement' -} as const; - -export const $ActionStatement_Output = { +export const $ActionStatement = { properties: { ref: { type: 'string', @@ -758,10 +655,15 @@ export const $DSLContext = { }, ENV: { '$ref': '#/components/schemas/DSLEnvironment' + }, + SECRETS: { + type: 'object', + title: 'Secrets' } }, type: 'object', - title: 'DSLContext' + title: 'DSLContext', + description: 'DSL Context. Contains all the context needed to execute a DSL workflow.' } as const; export const $DSLEntrypoint = { @@ -837,7 +739,7 @@ export const $DSLInput = { }, actions: { items: { - '$ref': '#/components/schemas/ActionStatement-Output' + '$ref': '#/components/schemas/ActionStatement' }, type: 'array', title: 'Actions' @@ -1118,7 +1020,7 @@ export const $EventGroup = { action_input: { anyOf: [ { - '$ref': '#/components/schemas/RunActionInput-Output' + '$ref': '#/components/schemas/RunActionInput' }, { '$ref': '#/components/schemas/DSLRunArgs' @@ -1333,7 +1235,7 @@ export const $GetWorkflowDefinitionActivityInputs = { task: { anyOf: [ { - '$ref': '#/components/schemas/ActionStatement-Output' + '$ref': '#/components/schemas/ActionStatement' }, { type: 'null' @@ -1868,18 +1770,6 @@ export const $RegistryActionUpdate = { description: 'API update model for a registered action.' } as const; -export const $RegistryActionValidate = { - properties: { - args: { - type: 'object', - title: 'Args' - } - }, - type: 'object', - required: ['args'], - title: 'RegistryActionValidate' -} as const; - export const $RegistryActionValidateResponse = { properties: { ok: { @@ -2086,7 +1976,7 @@ export const $Role = { }, service_id: { type: 'string', - enum: ['tracecat-runner', 'tracecat-api', 'tracecat-cli', 'tracecat-schedule-runner', 'tracecat-service'], + enum: ['tracecat-runner', 'tracecat-api', 'tracecat-cli', 'tracecat-schedule-runner', 'tracecat-service', 'tracecat-executor'], title: 'Service Id' } }, @@ -2119,28 +2009,10 @@ Service roles - A service's \`user_id\` is the user it's acting on behalf of. This can be None for internal services.` } as const; -export const $RunActionInput_Input = { - properties: { - task: { - '$ref': '#/components/schemas/ActionStatement-Input' - }, - exec_context: { - '$ref': '#/components/schemas/DSLContext' - }, - run_context: { - '$ref': '#/components/schemas/RunContext' - } - }, - type: 'object', - required: ['task', 'exec_context', 'run_context'], - title: 'RunActionInput', - description: 'This object contains all the information needed to execute an action.' -} as const; - -export const $RunActionInput_Output = { +export const $RunActionInput = { properties: { task: { - '$ref': '#/components/schemas/ActionStatement-Output' + '$ref': '#/components/schemas/ActionStatement' }, exec_context: { '$ref': '#/components/schemas/DSLContext' diff --git a/frontend/src/client/services.gen.ts b/frontend/src/client/services.gen.ts index e707bcc47..842c398a6 100644 --- a/frontend/src/client/services.gen.ts +++ b/frontend/src/client/services.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { PublicIncomingWebhookData, PublicIncomingWebhookResponse, PublicIncomingWebhookWaitData, PublicIncomingWebhookWaitResponse, WorkspacesListWorkspacesResponse, WorkspacesCreateWorkspaceData, WorkspacesCreateWorkspaceResponse, WorkspacesSearchWorkspacesData, WorkspacesSearchWorkspacesResponse, WorkspacesGetWorkspaceData, WorkspacesGetWorkspaceResponse, WorkspacesUpdateWorkspaceData, WorkspacesUpdateWorkspaceResponse, WorkspacesDeleteWorkspaceData, WorkspacesDeleteWorkspaceResponse, WorkspacesListWorkspaceMembershipsData, WorkspacesListWorkspaceMembershipsResponse, WorkspacesCreateWorkspaceMembershipData, WorkspacesCreateWorkspaceMembershipResponse, WorkspacesGetWorkspaceMembershipData, WorkspacesGetWorkspaceMembershipResponse, WorkspacesDeleteWorkspaceMembershipData, WorkspacesDeleteWorkspaceMembershipResponse, WorkflowsListWorkflowsData, WorkflowsListWorkflowsResponse, WorkflowsCreateWorkflowData, WorkflowsCreateWorkflowResponse, WorkflowsGetWorkflowData, WorkflowsGetWorkflowResponse, WorkflowsUpdateWorkflowData, WorkflowsUpdateWorkflowResponse, WorkflowsDeleteWorkflowData, WorkflowsDeleteWorkflowResponse, WorkflowsCommitWorkflowData, WorkflowsCommitWorkflowResponse, WorkflowsExportWorkflowData, WorkflowsExportWorkflowResponse, WorkflowsGetWorkflowDefinitionData, WorkflowsGetWorkflowDefinitionResponse, WorkflowsCreateWorkflowDefinitionData, WorkflowsCreateWorkflowDefinitionResponse, TriggersCreateWebhookData, TriggersCreateWebhookResponse, TriggersGetWebhookData, TriggersGetWebhookResponse, TriggersUpdateWebhookData, TriggersUpdateWebhookResponse, WorkflowExecutionsListWorkflowExecutionsData, WorkflowExecutionsListWorkflowExecutionsResponse, WorkflowExecutionsCreateWorkflowExecutionData, WorkflowExecutionsCreateWorkflowExecutionResponse, WorkflowExecutionsGetWorkflowExecutionData, WorkflowExecutionsGetWorkflowExecutionResponse, WorkflowExecutionsListWorkflowExecutionEventHistoryData, WorkflowExecutionsListWorkflowExecutionEventHistoryResponse, WorkflowExecutionsCancelWorkflowExecutionData, WorkflowExecutionsCancelWorkflowExecutionResponse, WorkflowExecutionsTerminateWorkflowExecutionData, WorkflowExecutionsTerminateWorkflowExecutionResponse, ActionsListActionsData, ActionsListActionsResponse, ActionsCreateActionData, ActionsCreateActionResponse, ActionsGetActionData, ActionsGetActionResponse, ActionsUpdateActionData, ActionsUpdateActionResponse, ActionsDeleteActionData, ActionsDeleteActionResponse, SecretsSearchSecretsData, SecretsSearchSecretsResponse, SecretsListSecretsData, SecretsListSecretsResponse, SecretsCreateSecretData, SecretsCreateSecretResponse, SecretsGetSecretByNameData, SecretsGetSecretByNameResponse, SecretsUpdateSecretByIdData, SecretsUpdateSecretByIdResponse, SecretsDeleteSecretByIdData, SecretsDeleteSecretByIdResponse, SchedulesListSchedulesData, SchedulesListSchedulesResponse, SchedulesCreateScheduleData, SchedulesCreateScheduleResponse, SchedulesGetScheduleData, SchedulesGetScheduleResponse, SchedulesUpdateScheduleData, SchedulesUpdateScheduleResponse, SchedulesDeleteScheduleData, SchedulesDeleteScheduleResponse, SchedulesSearchSchedulesData, SchedulesSearchSchedulesResponse, UsersSearchUserData, UsersSearchUserResponse, RegistryRepositoriesSyncRegistryRepositoriesData, RegistryRepositoriesSyncRegistryRepositoriesResponse, RegistryRepositoriesListRegistryRepositoriesResponse, RegistryRepositoriesCreateRegistryRepositoryData, RegistryRepositoriesCreateRegistryRepositoryResponse, RegistryRepositoriesGetRegistryRepositoryData, RegistryRepositoriesGetRegistryRepositoryResponse, RegistryRepositoriesUpdateRegistryRepositoryData, RegistryRepositoriesUpdateRegistryRepositoryResponse, RegistryRepositoriesDeleteRegistryRepositoryData, RegistryRepositoriesDeleteRegistryRepositoryResponse, RegistryActionsListRegistryActionsResponse, RegistryActionsCreateRegistryActionData, RegistryActionsCreateRegistryActionResponse, RegistryActionsGetRegistryActionData, RegistryActionsGetRegistryActionResponse, RegistryActionsUpdateRegistryActionData, RegistryActionsUpdateRegistryActionResponse, RegistryActionsDeleteRegistryActionData, RegistryActionsDeleteRegistryActionResponse, RegistryActionsRunRegistryActionData, RegistryActionsRunRegistryActionResponse, RegistryActionsValidateRegistryActionData, RegistryActionsValidateRegistryActionResponse, OrganizationListOrgMembersResponse, OrganizationDeleteOrgMemberData, OrganizationDeleteOrgMemberResponse, OrganizationUpdateOrgMemberData, OrganizationUpdateOrgMemberResponse, OrganizationListSessionsResponse, OrganizationDeleteSessionData, OrganizationDeleteSessionResponse, EditorListFunctionsData, EditorListFunctionsResponse, EditorListActionsData, EditorListActionsResponse, UsersUsersCurrentUserResponse, UsersUsersPatchCurrentUserData, UsersUsersPatchCurrentUserResponse, UsersUsersUserData, UsersUsersUserResponse, UsersUsersPatchUserData, UsersUsersPatchUserResponse, UsersUsersDeleteUserData, UsersUsersDeleteUserResponse, AuthAuthDatabaseLoginData, AuthAuthDatabaseLoginResponse, AuthAuthDatabaseLogoutResponse, AuthRegisterRegisterData, AuthRegisterRegisterResponse, AuthResetForgotPasswordData, AuthResetForgotPasswordResponse, AuthResetResetPasswordData, AuthResetResetPasswordResponse, AuthVerifyRequestTokenData, AuthVerifyRequestTokenResponse, AuthVerifyVerifyData, AuthVerifyVerifyResponse, AuthOauthGoogleDatabaseAuthorizeData, AuthOauthGoogleDatabaseAuthorizeResponse, AuthOauthGoogleDatabaseCallbackData, AuthOauthGoogleDatabaseCallbackResponse, AuthSamlDatabaseLoginResponse, AuthSsoAcsData, AuthSsoAcsResponse, PublicCheckHealthResponse } from './types.gen'; +import type { PublicIncomingWebhookData, PublicIncomingWebhookResponse, PublicIncomingWebhookWaitData, PublicIncomingWebhookWaitResponse, WorkspacesListWorkspacesResponse, WorkspacesCreateWorkspaceData, WorkspacesCreateWorkspaceResponse, WorkspacesSearchWorkspacesData, WorkspacesSearchWorkspacesResponse, WorkspacesGetWorkspaceData, WorkspacesGetWorkspaceResponse, WorkspacesUpdateWorkspaceData, WorkspacesUpdateWorkspaceResponse, WorkspacesDeleteWorkspaceData, WorkspacesDeleteWorkspaceResponse, WorkspacesListWorkspaceMembershipsData, WorkspacesListWorkspaceMembershipsResponse, WorkspacesCreateWorkspaceMembershipData, WorkspacesCreateWorkspaceMembershipResponse, WorkspacesGetWorkspaceMembershipData, WorkspacesGetWorkspaceMembershipResponse, WorkspacesDeleteWorkspaceMembershipData, WorkspacesDeleteWorkspaceMembershipResponse, WorkflowsListWorkflowsData, WorkflowsListWorkflowsResponse, WorkflowsCreateWorkflowData, WorkflowsCreateWorkflowResponse, WorkflowsGetWorkflowData, WorkflowsGetWorkflowResponse, WorkflowsUpdateWorkflowData, WorkflowsUpdateWorkflowResponse, WorkflowsDeleteWorkflowData, WorkflowsDeleteWorkflowResponse, WorkflowsCommitWorkflowData, WorkflowsCommitWorkflowResponse, WorkflowsExportWorkflowData, WorkflowsExportWorkflowResponse, WorkflowsGetWorkflowDefinitionData, WorkflowsGetWorkflowDefinitionResponse, WorkflowsCreateWorkflowDefinitionData, WorkflowsCreateWorkflowDefinitionResponse, TriggersCreateWebhookData, TriggersCreateWebhookResponse, TriggersGetWebhookData, TriggersGetWebhookResponse, TriggersUpdateWebhookData, TriggersUpdateWebhookResponse, WorkflowExecutionsListWorkflowExecutionsData, WorkflowExecutionsListWorkflowExecutionsResponse, WorkflowExecutionsCreateWorkflowExecutionData, WorkflowExecutionsCreateWorkflowExecutionResponse, WorkflowExecutionsGetWorkflowExecutionData, WorkflowExecutionsGetWorkflowExecutionResponse, WorkflowExecutionsListWorkflowExecutionEventHistoryData, WorkflowExecutionsListWorkflowExecutionEventHistoryResponse, WorkflowExecutionsCancelWorkflowExecutionData, WorkflowExecutionsCancelWorkflowExecutionResponse, WorkflowExecutionsTerminateWorkflowExecutionData, WorkflowExecutionsTerminateWorkflowExecutionResponse, ActionsListActionsData, ActionsListActionsResponse, ActionsCreateActionData, ActionsCreateActionResponse, ActionsGetActionData, ActionsGetActionResponse, ActionsUpdateActionData, ActionsUpdateActionResponse, ActionsDeleteActionData, ActionsDeleteActionResponse, SecretsSearchSecretsData, SecretsSearchSecretsResponse, SecretsListSecretsData, SecretsListSecretsResponse, SecretsCreateSecretData, SecretsCreateSecretResponse, SecretsGetSecretByNameData, SecretsGetSecretByNameResponse, SecretsUpdateSecretByIdData, SecretsUpdateSecretByIdResponse, SecretsDeleteSecretByIdData, SecretsDeleteSecretByIdResponse, SchedulesListSchedulesData, SchedulesListSchedulesResponse, SchedulesCreateScheduleData, SchedulesCreateScheduleResponse, SchedulesGetScheduleData, SchedulesGetScheduleResponse, SchedulesUpdateScheduleData, SchedulesUpdateScheduleResponse, SchedulesDeleteScheduleData, SchedulesDeleteScheduleResponse, SchedulesSearchSchedulesData, SchedulesSearchSchedulesResponse, UsersSearchUserData, UsersSearchUserResponse, OrganizationListOrgMembersResponse, OrganizationDeleteOrgMemberData, OrganizationDeleteOrgMemberResponse, OrganizationUpdateOrgMemberData, OrganizationUpdateOrgMemberResponse, OrganizationListSessionsResponse, OrganizationDeleteSessionData, OrganizationDeleteSessionResponse, EditorListFunctionsData, EditorListFunctionsResponse, EditorListActionsData, EditorListActionsResponse, RegistryRepositoriesSyncRegistryRepositoriesData, RegistryRepositoriesSyncRegistryRepositoriesResponse, RegistryRepositoriesListRegistryRepositoriesResponse, RegistryRepositoriesCreateRegistryRepositoryData, RegistryRepositoriesCreateRegistryRepositoryResponse, RegistryRepositoriesGetRegistryRepositoryData, RegistryRepositoriesGetRegistryRepositoryResponse, RegistryRepositoriesUpdateRegistryRepositoryData, RegistryRepositoriesUpdateRegistryRepositoryResponse, RegistryRepositoriesDeleteRegistryRepositoryData, RegistryRepositoriesDeleteRegistryRepositoryResponse, RegistryActionsListRegistryActionsResponse, RegistryActionsCreateRegistryActionData, RegistryActionsCreateRegistryActionResponse, RegistryActionsGetRegistryActionData, RegistryActionsGetRegistryActionResponse, RegistryActionsUpdateRegistryActionData, RegistryActionsUpdateRegistryActionResponse, RegistryActionsDeleteRegistryActionData, RegistryActionsDeleteRegistryActionResponse, UsersUsersCurrentUserResponse, UsersUsersPatchCurrentUserData, UsersUsersPatchCurrentUserResponse, UsersUsersUserData, UsersUsersUserResponse, UsersUsersPatchUserData, UsersUsersPatchUserResponse, UsersUsersDeleteUserData, UsersUsersDeleteUserResponse, AuthAuthDatabaseLoginData, AuthAuthDatabaseLoginResponse, AuthAuthDatabaseLogoutResponse, AuthRegisterRegisterData, AuthRegisterRegisterResponse, AuthResetForgotPasswordData, AuthResetForgotPasswordResponse, AuthResetResetPasswordData, AuthResetResetPasswordResponse, AuthVerifyRequestTokenData, AuthVerifyRequestTokenResponse, AuthVerifyVerifyData, AuthVerifyVerifyResponse, AuthOauthGoogleDatabaseAuthorizeData, AuthOauthGoogleDatabaseAuthorizeResponse, AuthOauthGoogleDatabaseCallbackData, AuthOauthGoogleDatabaseCallbackResponse, AuthSamlDatabaseLoginResponse, AuthSsoAcsData, AuthSsoAcsResponse, PublicCheckHealthResponse } from './types.gen'; /** * Incoming Webhook @@ -1110,6 +1110,121 @@ export const usersSearchUser = (data: UsersSearchUserData = {}): CancelablePromi } }); }; +/** + * List Org Members + * @returns OrgMemberRead Successful Response + * @throws ApiError + */ +export const organizationListOrgMembers = (): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/organization/members' +}); }; + +/** + * Delete Org Member + * @param data The data for the request. + * @param data.userId + * @returns void Successful Response + * @throws ApiError + */ +export const organizationDeleteOrgMember = (data: OrganizationDeleteOrgMemberData): CancelablePromise => { return __request(OpenAPI, { + method: 'DELETE', + url: '/organization/members/{user_id}', + path: { + user_id: data.userId + }, + errors: { + 422: 'Validation Error' + } +}); }; + +/** + * Update Org Member + * @param data The data for the request. + * @param data.userId + * @param data.requestBody + * @returns OrgMemberRead Successful Response + * @throws ApiError + */ +export const organizationUpdateOrgMember = (data: OrganizationUpdateOrgMemberData): CancelablePromise => { return __request(OpenAPI, { + method: 'PATCH', + url: '/organization/members/{user_id}', + path: { + user_id: data.userId + }, + body: data.requestBody, + mediaType: 'application/json', + errors: { + 422: 'Validation Error' + } +}); }; + +/** + * List Sessions + * @returns SessionRead Successful Response + * @throws ApiError + */ +export const organizationListSessions = (): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/organization/sessions' +}); }; + +/** + * Delete Session + * @param data The data for the request. + * @param data.sessionId + * @returns void Successful Response + * @throws ApiError + */ +export const organizationDeleteSession = (data: OrganizationDeleteSessionData): CancelablePromise => { return __request(OpenAPI, { + method: 'DELETE', + url: '/organization/sessions/{session_id}', + path: { + session_id: data.sessionId + }, + errors: { + 422: 'Validation Error' + } +}); }; + +/** + * List Functions + * @param data The data for the request. + * @param data.workspaceId + * @returns EditorFunctionRead Successful Response + * @throws ApiError + */ +export const editorListFunctions = (data: EditorListFunctionsData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/editor/functions', + query: { + workspace_id: data.workspaceId + }, + errors: { + 422: 'Validation Error' + } +}); }; + +/** + * List Actions + * @param data The data for the request. + * @param data.workflowId + * @param data.workspaceId + * @returns EditorActionRead Successful Response + * @throws ApiError + */ +export const editorListActions = (data: EditorListActionsData): CancelablePromise => { return __request(OpenAPI, { + method: 'GET', + url: '/editor/actions', + query: { + workflow_id: data.workflowId, + workspace_id: data.workspaceId + }, + errors: { + 422: 'Validation Error' + } +}); }; + /** * Sync Registry Repositories * Load actions from all registry repositories. @@ -1307,165 +1422,6 @@ export const registryActionsDeleteRegistryAction = (data: RegistryActionsDeleteR } }); }; -/** - * Run Registry Action - * Execute a registry action. - * @param data The data for the request. - * @param data.actionName - * @param data.requestBody - * @returns unknown Successful Response - * @throws ApiError - */ -export const registryActionsRunRegistryAction = (data: RegistryActionsRunRegistryActionData): CancelablePromise => { return __request(OpenAPI, { - method: 'POST', - url: '/registry/actions/{action_name}/execute', - path: { - action_name: data.actionName - }, - body: data.requestBody, - mediaType: 'application/json', - errors: { - 422: 'Validation Error' - } -}); }; - -/** - * Validate Registry Action - * Validate a registry action. - * @param data The data for the request. - * @param data.actionName - * @param data.requestBody - * @returns RegistryActionValidateResponse Successful Response - * @throws ApiError - */ -export const registryActionsValidateRegistryAction = (data: RegistryActionsValidateRegistryActionData): CancelablePromise => { return __request(OpenAPI, { - method: 'POST', - url: '/registry/actions/{action_name}/validate', - path: { - action_name: data.actionName - }, - body: data.requestBody, - mediaType: 'application/json', - errors: { - 422: 'Validation Error' - } -}); }; - -/** - * List Org Members - * @returns OrgMemberRead Successful Response - * @throws ApiError - */ -export const organizationListOrgMembers = (): CancelablePromise => { return __request(OpenAPI, { - method: 'GET', - url: '/organization/members' -}); }; - -/** - * Delete Org Member - * @param data The data for the request. - * @param data.userId - * @returns void Successful Response - * @throws ApiError - */ -export const organizationDeleteOrgMember = (data: OrganizationDeleteOrgMemberData): CancelablePromise => { return __request(OpenAPI, { - method: 'DELETE', - url: '/organization/members/{user_id}', - path: { - user_id: data.userId - }, - errors: { - 422: 'Validation Error' - } -}); }; - -/** - * Update Org Member - * @param data The data for the request. - * @param data.userId - * @param data.requestBody - * @returns OrgMemberRead Successful Response - * @throws ApiError - */ -export const organizationUpdateOrgMember = (data: OrganizationUpdateOrgMemberData): CancelablePromise => { return __request(OpenAPI, { - method: 'PATCH', - url: '/organization/members/{user_id}', - path: { - user_id: data.userId - }, - body: data.requestBody, - mediaType: 'application/json', - errors: { - 422: 'Validation Error' - } -}); }; - -/** - * List Sessions - * @returns SessionRead Successful Response - * @throws ApiError - */ -export const organizationListSessions = (): CancelablePromise => { return __request(OpenAPI, { - method: 'GET', - url: '/organization/sessions' -}); }; - -/** - * Delete Session - * @param data The data for the request. - * @param data.sessionId - * @returns void Successful Response - * @throws ApiError - */ -export const organizationDeleteSession = (data: OrganizationDeleteSessionData): CancelablePromise => { return __request(OpenAPI, { - method: 'DELETE', - url: '/organization/sessions/{session_id}', - path: { - session_id: data.sessionId - }, - errors: { - 422: 'Validation Error' - } -}); }; - -/** - * List Functions - * @param data The data for the request. - * @param data.workspaceId - * @returns EditorFunctionRead Successful Response - * @throws ApiError - */ -export const editorListFunctions = (data: EditorListFunctionsData): CancelablePromise => { return __request(OpenAPI, { - method: 'GET', - url: '/editor/functions', - query: { - workspace_id: data.workspaceId - }, - errors: { - 422: 'Validation Error' - } -}); }; - -/** - * List Actions - * @param data The data for the request. - * @param data.workflowId - * @param data.workspaceId - * @returns EditorActionRead Successful Response - * @throws ApiError - */ -export const editorListActions = (data: EditorListActionsData): CancelablePromise => { return __request(OpenAPI, { - method: 'GET', - url: '/editor/actions', - query: { - workflow_id: data.workflowId, - workspace_id: data.workspaceId - }, - errors: { - 422: 'Validation Error' - } -}); }; - /** * Users:Current User * @returns UserRead Successful Response diff --git a/frontend/src/client/types.gen.ts b/frontend/src/client/types.gen.ts index c1a6492e2..27186f3f0 100644 --- a/frontend/src/client/types.gen.ts +++ b/frontend/src/client/types.gen.ts @@ -56,53 +56,7 @@ export type ActionRetryPolicy = { timeout?: number; }; -export type ActionStatement_Input = { - /** - * The action ID. If this is populated means there is a corresponding actionin the database `Action` table. - */ - id?: string | null; - /** - * Unique reference for the task - */ - ref: string; - description?: string; - /** - * Action type. Equivalent to the UDF key. - */ - action: string; - /** - * Arguments for the action - */ - args?: { - [key: string]: unknown; - }; - /** - * Task dependencies - */ - depends_on?: Array<(string)>; - /** - * Condition to run the task - */ - run_if?: string | null; - /** - * Iterate over a list of items and run the task for each item. - */ - for_each?: string | Array<(string)> | null; - /** - * Retry policy for the action. - */ - retry_policy?: ActionRetryPolicy; - /** - * Delay before starting the action in seconds. - */ - start_delay?: number; - /** - * The strategy to use when joining on this task. By default, all branches must complete successfully before the join task can complete. - */ - join_strategy?: JoinStrategy; -}; - -export type ActionStatement_Output = { +export type ActionStatement = { /** * Unique reference for the task */ @@ -268,6 +222,9 @@ export type DSLConfig_Output = { timeout?: number; }; +/** + * DSL Context. Contains all the context needed to execute a DSL workflow. + */ export type DSLContext = { INPUTS?: { [key: string]: unknown; @@ -277,6 +234,9 @@ export type DSLContext = { }; TRIGGER?: JsonValue; ENV?: DSLEnvironment; + SECRETS?: { + [key: string]: unknown; + }; }; export type DSLEntrypoint = { @@ -320,7 +280,7 @@ export type DSLInput = { title: string; description: string; entrypoint: DSLEntrypoint; - actions: Array; + actions: Array; config?: DSLConfig_Output; triggers?: Array; /** @@ -400,7 +360,7 @@ export type EventGroup = { action_ref: string; action_title: string; action_description: string; - action_input: RunActionInput_Output | DSLRunArgs | GetWorkflowDefinitionActivityInputs; + action_input: RunActionInput | DSLRunArgs | GetWorkflowDefinitionActivityInputs; action_result?: unknown | null; current_attempt?: number | null; retry_policy?: ActionRetryPolicy; @@ -440,7 +400,7 @@ export type GetWorkflowDefinitionActivityInputs = { role: Role; workflow_id: string; version?: number | null; - task?: ActionStatement_Output | null; + task?: ActionStatement | null; }; export type HTTPValidationError = { @@ -657,12 +617,6 @@ export type RegistryActionUpdate = { options?: RegistryActionOptions | null; }; -export type RegistryActionValidate = { - args: { - [key: string]: unknown; - }; -}; - export type RegistryActionValidateResponse = { ok: boolean; message: string; @@ -729,27 +683,18 @@ export type Role = { workspace_id?: string | null; user_id?: string | null; access_level?: AccessLevel; - service_id: 'tracecat-runner' | 'tracecat-api' | 'tracecat-cli' | 'tracecat-schedule-runner' | 'tracecat-service'; + service_id: 'tracecat-runner' | 'tracecat-api' | 'tracecat-cli' | 'tracecat-schedule-runner' | 'tracecat-service' | 'tracecat-executor'; }; export type type2 = 'user' | 'service'; -export type service_id = 'tracecat-runner' | 'tracecat-api' | 'tracecat-cli' | 'tracecat-schedule-runner' | 'tracecat-service'; +export type service_id = 'tracecat-runner' | 'tracecat-api' | 'tracecat-cli' | 'tracecat-schedule-runner' | 'tracecat-service' | 'tracecat-executor'; /** * This object contains all the information needed to execute an action. */ -export type RunActionInput_Input = { - task: ActionStatement_Input; - exec_context: DSLContext; - run_context: RunContext; -}; - -/** - * This object contains all the information needed to execute an action. - */ -export type RunActionInput_Output = { - task: ActionStatement_Output; +export type RunActionInput = { + task: ActionStatement; exec_context: DSLContext; run_context: RunContext; }; @@ -1626,6 +1571,42 @@ export type UsersSearchUserData = { export type UsersSearchUserResponse = UserRead; +export type OrganizationListOrgMembersResponse = Array; + +export type OrganizationDeleteOrgMemberData = { + userId: string; +}; + +export type OrganizationDeleteOrgMemberResponse = void; + +export type OrganizationUpdateOrgMemberData = { + requestBody: UserUpdate; + userId: string; +}; + +export type OrganizationUpdateOrgMemberResponse = OrgMemberRead; + +export type OrganizationListSessionsResponse = Array; + +export type OrganizationDeleteSessionData = { + sessionId: string; +}; + +export type OrganizationDeleteSessionResponse = void; + +export type EditorListFunctionsData = { + workspaceId: string; +}; + +export type EditorListFunctionsResponse = Array; + +export type EditorListActionsData = { + workflowId: string; + workspaceId: string; +}; + +export type EditorListActionsResponse = Array; + export type RegistryRepositoriesSyncRegistryRepositoriesData = { /** * Origins to sync. If no origins provided, all repositories will be synced. @@ -1689,56 +1670,6 @@ export type RegistryActionsDeleteRegistryActionData = { export type RegistryActionsDeleteRegistryActionResponse = void; -export type RegistryActionsRunRegistryActionData = { - actionName: string; - requestBody: RunActionInput_Input; -}; - -export type RegistryActionsRunRegistryActionResponse = unknown; - -export type RegistryActionsValidateRegistryActionData = { - actionName: string; - requestBody: RegistryActionValidate; -}; - -export type RegistryActionsValidateRegistryActionResponse = RegistryActionValidateResponse; - -export type OrganizationListOrgMembersResponse = Array; - -export type OrganizationDeleteOrgMemberData = { - userId: string; -}; - -export type OrganizationDeleteOrgMemberResponse = void; - -export type OrganizationUpdateOrgMemberData = { - requestBody: UserUpdate; - userId: string; -}; - -export type OrganizationUpdateOrgMemberResponse = OrgMemberRead; - -export type OrganizationListSessionsResponse = Array; - -export type OrganizationDeleteSessionData = { - sessionId: string; -}; - -export type OrganizationDeleteSessionResponse = void; - -export type EditorListFunctionsData = { - workspaceId: string; -}; - -export type EditorListFunctionsResponse = Array; - -export type EditorListActionsData = { - workflowId: string; - workspaceId: string; -}; - -export type EditorListActionsResponse = Array; - export type UsersUsersCurrentUserResponse = UserRead; export type UsersUsersPatchCurrentUserData = { @@ -2507,75 +2438,57 @@ export type $OpenApiTs = { }; }; }; - '/registry/repos/sync': { - post: { - req: RegistryRepositoriesSyncRegistryRepositoriesData; - res: { - /** - * Successful Response - */ - 204: void; - /** - * Validation Error - */ - 422: HTTPValidationError; - }; - }; - }; - '/registry/repos': { + '/organization/members': { get: { res: { /** * Successful Response */ - 200: Array; + 200: Array; }; }; - post: { - req: RegistryRepositoriesCreateRegistryRepositoryData; + }; + '/organization/members/{user_id}': { + delete: { + req: OrganizationDeleteOrgMemberData; res: { /** * Successful Response */ - 201: RegistryRepositoryRead; + 204: void; /** * Validation Error */ 422: HTTPValidationError; }; }; - }; - '/registry/repos/{origin}': { - get: { - req: RegistryRepositoriesGetRegistryRepositoryData; + patch: { + req: OrganizationUpdateOrgMemberData; res: { /** * Successful Response */ - 200: RegistryRepositoryRead; + 200: OrgMemberRead; /** * Validation Error */ 422: HTTPValidationError; }; }; - patch: { - req: RegistryRepositoriesUpdateRegistryRepositoryData; + }; + '/organization/sessions': { + get: { res: { /** * Successful Response */ - 200: RegistryRepositoryRead; - /** - * Validation Error - */ - 422: HTTPValidationError; + 200: Array; }; }; }; - '/registry/repos/{id}': { + '/organization/sessions/{session_id}': { delete: { - req: RegistryRepositoriesDeleteRegistryRepositoryData; + req: OrganizationDeleteSessionData; res: { /** * Successful Response @@ -2588,22 +2501,14 @@ export type $OpenApiTs = { }; }; }; - '/registry/actions': { + '/editor/functions': { get: { + req: EditorListFunctionsData; res: { /** * Successful Response */ - 200: Array; - }; - }; - post: { - req: RegistryActionsCreateRegistryActionData; - res: { - /** - * Successful Response - */ - 201: RegistryActionRead; + 200: Array; /** * Validation Error */ @@ -2611,22 +2516,24 @@ export type $OpenApiTs = { }; }; }; - '/registry/actions/{action_name}': { + '/editor/actions': { get: { - req: RegistryActionsGetRegistryActionData; + req: EditorListActionsData; res: { /** * Successful Response */ - 200: RegistryActionRead; + 200: Array; /** * Validation Error */ 422: HTTPValidationError; }; }; - patch: { - req: RegistryActionsUpdateRegistryActionData; + }; + '/registry/repos/sync': { + post: { + req: RegistryRepositoriesSyncRegistryRepositoriesData; res: { /** * Successful Response @@ -2638,28 +2545,23 @@ export type $OpenApiTs = { 422: HTTPValidationError; }; }; - delete: { - req: RegistryActionsDeleteRegistryActionData; + }; + '/registry/repos': { + get: { res: { /** * Successful Response */ - 204: void; - /** - * Validation Error - */ - 422: HTTPValidationError; + 200: Array; }; }; - }; - '/registry/actions/{action_name}/execute': { post: { - req: RegistryActionsRunRegistryActionData; + req: RegistryRepositoriesCreateRegistryRepositoryData; res: { /** * Successful Response */ - 200: unknown; + 201: RegistryRepositoryRead; /** * Validation Error */ @@ -2667,34 +2569,37 @@ export type $OpenApiTs = { }; }; }; - '/registry/actions/{action_name}/validate': { - post: { - req: RegistryActionsValidateRegistryActionData; + '/registry/repos/{origin}': { + get: { + req: RegistryRepositoriesGetRegistryRepositoryData; res: { /** * Successful Response */ - 200: RegistryActionValidateResponse; + 200: RegistryRepositoryRead; /** * Validation Error */ 422: HTTPValidationError; }; }; - }; - '/organization/members': { - get: { + patch: { + req: RegistryRepositoriesUpdateRegistryRepositoryData; res: { /** * Successful Response */ - 200: Array; + 200: RegistryRepositoryRead; + /** + * Validation Error + */ + 422: HTTPValidationError; }; }; }; - '/organization/members/{user_id}': { + '/registry/repos/{id}': { delete: { - req: OrganizationDeleteOrgMemberData; + req: RegistryRepositoriesDeleteRegistryRepositoryData; res: { /** * Successful Response @@ -2706,68 +2611,64 @@ export type $OpenApiTs = { 422: HTTPValidationError; }; }; - patch: { - req: OrganizationUpdateOrgMemberData; + }; + '/registry/actions': { + get: { res: { /** * Successful Response */ - 200: OrgMemberRead; - /** - * Validation Error - */ - 422: HTTPValidationError; + 200: Array; }; }; - }; - '/organization/sessions': { - get: { + post: { + req: RegistryActionsCreateRegistryActionData; res: { /** * Successful Response */ - 200: Array; + 201: RegistryActionRead; + /** + * Validation Error + */ + 422: HTTPValidationError; }; }; }; - '/organization/sessions/{session_id}': { - delete: { - req: OrganizationDeleteSessionData; + '/registry/actions/{action_name}': { + get: { + req: RegistryActionsGetRegistryActionData; res: { /** * Successful Response */ - 204: void; + 200: RegistryActionRead; /** * Validation Error */ 422: HTTPValidationError; }; }; - }; - '/editor/functions': { - get: { - req: EditorListFunctionsData; + patch: { + req: RegistryActionsUpdateRegistryActionData; res: { /** * Successful Response */ - 200: Array; + 204: void; /** * Validation Error */ 422: HTTPValidationError; }; }; - }; - '/editor/actions': { - get: { - req: EditorListActionsData; + delete: { + req: RegistryActionsDeleteRegistryActionData; res: { /** * Successful Response */ - 200: Array; + 204: void; /** * Validation Error */ diff --git a/frontend/src/components/executions/event-details.tsx b/frontend/src/components/executions/event-details.tsx index 5bd323b35..7b3263986 100644 --- a/frontend/src/components/executions/event-details.tsx +++ b/frontend/src/components/executions/event-details.tsx @@ -1,9 +1,5 @@ import React from "react" -import { - DSLRunArgs, - EventHistoryResponse, - RunActionInput_Output, -} from "@/client" +import { DSLRunArgs, EventHistoryResponse, RunActionInput } from "@/client" import JsonView from "react18-json-view" import { @@ -139,7 +135,7 @@ export function WorkflowExecutionEventDetailView({
- +
@@ -412,7 +408,7 @@ function ActionEventGeneralInfo({ task: { depends_on, run_if, for_each }, }, }: { - input: RunActionInput_Output + input: RunActionInput }) { return (
@@ -467,12 +463,12 @@ function ActionEventGeneralInfo({ function isRunActionInput_Output( actionInput: unknown -): actionInput is RunActionInput_Output { +): actionInput is RunActionInput { return ( typeof actionInput === "object" && actionInput !== null && "task" in actionInput && - typeof (actionInput as RunActionInput_Output).task === "object" + typeof (actionInput as RunActionInput).task === "object" ) } diff --git a/tests/conftest.py b/tests/conftest.py index ccb4072c3..e2993808e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,8 +59,9 @@ def env_sandbox(monkeysession: pytest.MonkeyPatch): "TRACECAT__REMOTE_REPOSITORY_URL", "git+ssh://git@github.com/TracecatHQ/udfs.git", ) + # Need this for local unit tests monkeysession.setattr( - config, "TRACECAT__REGISTRY_URL", "http://localhost/api/registry" + config, "TRACECAT__EXECUTOR_URL", "http://localhost/api/executor" ) monkeysession.setenv( @@ -69,7 +70,8 @@ def env_sandbox(monkeysession: pytest.MonkeyPatch): ) # monkeysession.setenv("TRACECAT__DB_ENCRYPTION_KEY", Fernet.generate_key().decode()) monkeysession.setenv("TRACECAT__API_URL", "http://api:8000") - monkeysession.setenv("TRACECAT__REGISTRY_URL", "http://registry:8000") + # Needed for local unit tests + monkeysession.setenv("TRACECAT__EXECUTOR_URL", "http://executor:8000") monkeysession.setenv("TRACECAT__PUBLIC_API_URL", "http://localhost/api") monkeysession.setenv("TRACECAT__PUBLIC_RUNNER_URL", "http://localhost:8001") monkeysession.setenv("TRACECAT__SERVICE_KEY", os.environ["TRACECAT__SERVICE_KEY"]) diff --git a/tests/unit/test_workflows.py b/tests/unit/test_workflows.py index 9f7a9eb46..1f35e4ab9 100644 --- a/tests/unit/test_workflows.py +++ b/tests/unit/test_workflows.py @@ -1601,7 +1601,7 @@ async def test_pull_based_workflow_fetches_latest_version(temporal_client, test_ "------------------------------\n" "File: /app/tracecat/registry/executor.py\n" "Function: run_action_in_pool\n" - "Line: 83" + "Line: 200" ), "type": "RegistryActionError", "expr_context": "ACTIONS", diff --git a/tracecat/api/app.py b/tracecat/api/app.py index 2832b06b7..0c678db62 100644 --- a/tracecat/api/app.py +++ b/tracecat/api/app.py @@ -12,6 +12,7 @@ from tracecat.api.common import ( custom_generate_unique_id, generic_exception_handler, + setup_registry, tracecat_exception_handler, ) from tracecat.auth.constants import AuthType @@ -29,6 +30,8 @@ from tracecat.logger import logger from tracecat.middleware import RequestLoggingMiddleware from tracecat.organization.router import router as org_router +from tracecat.registry.actions.router import router as registry_actions_router +from tracecat.registry.repositories.router import router as registry_repos_router from tracecat.secrets.router import router as secrets_router from tracecat.types.auth import AccessLevel, Role from tracecat.types.exceptions import TracecatException @@ -50,6 +53,7 @@ async def lifespan(app: FastAPI): ) async with get_async_session_context_manager() as session: await setup_defaults(session, admin_role) + await setup_registry(session, admin_role) yield @@ -139,6 +143,8 @@ def create_app(**kwargs) -> FastAPI: app.include_router(users_router) app.include_router(org_router) app.include_router(editor_router) + app.include_router(registry_repos_router) + app.include_router(registry_actions_router) app.include_router( fastapi_users.get_users_router(UserRead, UserUpdate), prefix="/users", diff --git a/tracecat/api/registry.py b/tracecat/api/executor.py similarity index 64% rename from tracecat/api/registry.py rename to tracecat/api/executor.py index 796955803..8664d2b61 100644 --- a/tracecat/api/registry.py +++ b/tracecat/api/executor.py @@ -9,31 +9,19 @@ custom_generate_unique_id, generic_exception_handler, setup_oss_models, - setup_registry, tracecat_exception_handler, ) -from tracecat.db.engine import get_async_session_context_manager from tracecat.logger import logger from tracecat.middleware import RequestLoggingMiddleware -from tracecat.registry.actions.router import router as registry_actions_router -from tracecat.registry.executor import get_executor -from tracecat.registry.repositories.router import router as registry_repos_router -from tracecat.types.auth import AccessLevel, Role +from tracecat.registry.executor import get_executor, router from tracecat.types.exceptions import TracecatException @asynccontextmanager async def lifespan(app: FastAPI): - admin_role = Role( - type="service", - access_level=AccessLevel.ADMIN, - service_id="tracecat-registry", - ) - async with get_async_session_context_manager() as session: - await setup_registry(session, admin_role) await setup_oss_models() + executor = get_executor() try: - executor = get_executor() yield finally: executor.shutdown() @@ -45,20 +33,19 @@ def create_app(**kwargs) -> FastAPI: else: allow_origins = ["*"] app = FastAPI( - title="Tracecat Registry", - description="Registry action executor.", - summary="Tracecat Registry", + title="Tracecat Executor", + description="Action executor for Tracecat.", + summary="Tracecat Executor", lifespan=lifespan, default_response_class=ORJSONResponse, generate_unique_id_function=custom_generate_unique_id, - root_path="/api/registry", + root_path="/api/executor", **kwargs, ) app.logger = logger # type: ignore # Routers - app.include_router(registry_repos_router) - app.include_router(registry_actions_router) + app.include_router(router) # Exception handlers app.add_exception_handler(Exception, generic_exception_handler) @@ -75,7 +62,7 @@ def create_app(**kwargs) -> FastAPI: ) logger.info( - "Registry service started", + "Executor service started", env=config.TRACECAT__APP_ENV, origins=allow_origins, auth_types=config.TRACECAT__AUTH_TYPES, @@ -89,4 +76,4 @@ def create_app(**kwargs) -> FastAPI: @app.get("/", include_in_schema=False) def root() -> dict[str, str]: - return {"message": "Hello world. I am the registry."} + return {"message": "Hello world. I am the executor."} diff --git a/tracecat/config.py b/tracecat/config.py index 07507ef63..443e72e62 100644 --- a/tracecat/config.py +++ b/tracecat/config.py @@ -32,8 +32,8 @@ "TRACECAT__DB_URI", "postgresql+psycopg://postgres:postgres@postgres_db:5432/postgres", ) -TRACECAT__REGISTRY_URL = os.environ.get( - "TRACECAT__REGISTRY_URL", "http://registry:8000" +TRACECAT__EXECUTOR_URL = os.environ.get( + "TRACECAT__EXECUTOR_URL", "http://executor:8000" ) TRACECAT__DB_NAME = os.environ.get("TRACECAT__DB_NAME") diff --git a/tracecat/identifiers/__init__.py b/tracecat/identifiers/__init__.py index d883a4778..19e274f7f 100644 --- a/tracecat/identifiers/__init__.py +++ b/tracecat/identifiers/__init__.py @@ -71,7 +71,7 @@ "tracecat-cli", "tracecat-schedule-runner", "tracecat-service", - "tracecat-registry", + "tracecat-executor", ] __all__ = [ diff --git a/tracecat/registry/actions/router.py b/tracecat/registry/actions/router.py index 8af7da1be..dc7f52484 100644 --- a/tracecat/registry/actions/router.py +++ b/tracecat/registry/actions/router.py @@ -1,29 +1,19 @@ -import traceback -from typing import Any - from fastapi import APIRouter, HTTPException, status from sqlalchemy.exc import IntegrityError from tracecat.auth.credentials import RoleACL from tracecat.concurrency import GatheringTaskGroup -from tracecat.contexts import ctx_logger, ctx_role from tracecat.db.dependencies import AsyncDBSession -from tracecat.dsl.models import RunActionInput from tracecat.logger import logger -from tracecat.registry import executor from tracecat.registry.actions.models import ( RegistryActionCreate, - RegistryActionErrorInfo, RegistryActionRead, RegistryActionUpdate, - RegistryActionValidate, - RegistryActionValidateResponse, ) from tracecat.registry.actions.service import RegistryActionsService from tracecat.registry.constants import DEFAULT_REGISTRY_ORIGIN, REGISTRY_ACTIONS_PATH from tracecat.types.auth import AccessLevel, Role from tracecat.types.exceptions import RegistryError -from tracecat.validation.service import validate_registry_action_args router = APIRouter(prefix=REGISTRY_ACTIONS_PATH, tags=["registry-actions"]) @@ -154,82 +144,3 @@ async def delete_registry_action( ) # Delete the action as it's not a base action await service.delete_action(action) - - -# Registry Action Controls - - -@router.post("/{action_name}/execute") -async def run_registry_action( - *, - role: Role = RoleACL( - allow_user=False, # XXX(authz): Users cannot execute actions - allow_service=True, # Only services can execute actions - require_workspace="no", - ), - action_name: str, - action_input: RunActionInput, -) -> Any: - """Execute a registry action.""" - ref = action_input.task.ref - ctx_role.set(role) - act_logger = logger.bind(role=role, action_name=action_name, ref=ref) - ctx_logger.set(act_logger) - - act_logger.info("Starting action") - try: - return await executor.run_action_in_pool(input=action_input) - except Exception as e: - # Get the traceback info - tb = traceback.extract_tb(e.__traceback__)[-1] # Get the last frame - error_detail = RegistryActionErrorInfo( - action_name=action_name, - type=e.__class__.__name__, - message=str(e), - filename=tb.filename, - function=tb.name, - lineno=tb.lineno, - ) - act_logger.error( - "Error running action", - action_name=action_name, - type=error_detail.type, - message=error_detail.message, - filename=error_detail.filename, - function=error_detail.function, - lineno=error_detail.lineno, - ) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=error_detail.model_dump(mode="json"), - ) from e - - -@router.post("/{action_name}/validate") -async def validate_registry_action( - *, - role: Role = RoleACL( - allow_user=False, # XXX(authz): Users cannot validate actions - allow_service=True, # Only services can validate actions - require_workspace="no", - ), - session: AsyncDBSession, - action_name: str, - params: RegistryActionValidate, -) -> RegistryActionValidateResponse: - """Validate a registry action.""" - try: - result = await validate_registry_action_args( - session=session, action_name=action_name, args=params.args - ) - - if result.status == "error": - logger.warning( - "Error validating UDF args", message=result.msg, details=result.detail - ) - return RegistryActionValidateResponse.from_validation_result(result) - except KeyError as e: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Action {action_name!r} not found in registry", - ) from e diff --git a/tracecat/registry/actions/service.py b/tracecat/registry/actions/service.py index 07a7402c0..5648059cc 100644 --- a/tracecat/registry/actions/service.py +++ b/tracecat/registry/actions/service.py @@ -22,6 +22,7 @@ RegistryActionUpdate, model_converters, ) +from tracecat.registry.client import RegistryClient from tracecat.registry.loaders import get_bound_action_impl from tracecat.registry.repository import Repository from tracecat.types.auth import Role @@ -208,9 +209,11 @@ async def sync_actions_from_repository(self, db_repo: RegistryRepository) -> Non - For each repository, we need to reimport the packages to run decorators. (for remote this involves pulling) - Scan the repositories for implementation details/metadata and update the DB """ + # (1) Update the API's view of the repository repo = Repository(origin=db_repo.origin, role=self.role) await repo.load_from_origin() + # (2) Handle DB bookkeeping for the API's view of the repository # Perform diffing here. The expectation for this endpoint is to sync Tracecat's view of # the repository with the remote repository -- meaning any creation/updates/deletions to # actions should be propogated to the db. @@ -275,6 +278,11 @@ async def sync_actions_from_repository(self, db_repo: RegistryRepository) -> Non deleted=n_deleted, ) + # (3) Update the executor's view of the repository + self.logger.info("Syncing executor", origin=db_repo.origin) + client = RegistryClient(role=self.role) + await client.sync_executor(origin=db_repo.origin) + async def load_action_impl(self, action_name: str) -> BoundRegistryAction: """ Load the implementation for a registry action. diff --git a/tracecat/registry/client.py b/tracecat/registry/client.py index 43bc50749..c6fa04039 100644 --- a/tracecat/registry/client.py +++ b/tracecat/registry/client.py @@ -1,11 +1,18 @@ """Use this in worker to execute actions.""" -from collections.abc import Mapping +from collections.abc import AsyncIterator, Mapping +from contextlib import asynccontextmanager from json import JSONDecodeError from typing import Any, cast import httpx import orjson +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) from tracecat import config from tracecat.clients import AuthenticatedServiceClient @@ -26,9 +33,11 @@ class _RegistryHTTPClient(AuthenticatedServiceClient): """Async httpx client for the registry service.""" def __init__(self, role: Role | None = None, *args: Any, **kwargs: Any) -> None: - self._registry_base_url = config.TRACECAT__REGISTRY_URL + self._registry_base_url = config.TRACECAT__EXECUTOR_URL super().__init__(role, *args, base_url=self._registry_base_url, **kwargs) - self.params = self.params.add("workspace_id", str(self.role.workspace_id)) + self.params = self.params.add( + "workspace_id", str(self.role.workspace_id) if self.role else None + ) class RegistryClient: @@ -42,6 +51,11 @@ def __init__(self, role: Role | None = None): self.role = role or ctx_role.get() self.logger = logger.bind(service="registry-client", role=self.role) + @asynccontextmanager + async def _client(self) -> AsyncIterator[_RegistryHTTPClient]: + async with _RegistryHTTPClient(self.role) as client: + yield client + """Execution""" async def call_action(self, input: RunActionInput) -> Any: @@ -75,7 +89,6 @@ async def call_action(self, input: RunActionInput) -> Any: action_type = input.task.action content = input.model_dump_json() - workspace_id = str(self.role.workspace_id) if self.role.workspace_id else None logger.debug( f"Calling action {action_type!r} with content", content=content, @@ -83,9 +96,9 @@ async def call_action(self, input: RunActionInput) -> Any: timeout=self._timeout, ) try: - async with _RegistryHTTPClient(self.role) as client: + async with self._client() as client: response = await client.post( - f"{self._actions_endpoint}/{action_type}/execute", + f"/run/{action_type}", # NOTE(perf): Maybe serialize with orjson.dumps instead headers={ "Content-Type": "application/json", @@ -93,7 +106,6 @@ async def call_action(self, input: RunActionInput) -> Any: **self.role.to_headers(), }, content=content, - params={"workspace_id": workspace_id}, timeout=self._timeout, ) response.raise_for_status() @@ -144,10 +156,9 @@ async def validate_action( """Validate an action.""" try: logger.warning("Validating action") - async with _RegistryHTTPClient(self.role) as client: + async with self._client() as client: response = await client.post( - f"{self._actions_endpoint}/{action_name}/validate", - json={"args": args}, + f"/validate/{action_name}", json={"args": args} ) response.raise_for_status() return RegistryActionValidateResponse.model_validate_json(response.content) @@ -164,6 +175,55 @@ async def validate_action( f"Unexpected error while listing registries: {str(e)}" ) from e + """Executor""" + + async def sync_executor(self, origin: str, *, max_attempts: int = 3) -> None: + """Sync the executor from the registry. + + Args: + origin: The origin of the sync request + + Raises: + RegistryError: If the sync fails after all retries + """ + + @retry( + stop=stop_after_attempt(max_attempts), + wait=wait_exponential(multiplier=1, min=4, max=10), + retry=retry_if_exception_type( + ( + httpx.HTTPStatusError, + httpx.RequestError, + httpx.TimeoutException, + httpx.ConnectError, + ) + ), + ) + async def _sync_request() -> None: + try: + async with self._client() as client: + response = await client.post("/sync", json={"origin": origin}) + response.raise_for_status() + except Exception as e: + logger.error("Error syncing executor", error=e) + raise + + try: + logger.info("Syncing executor", origin=origin) + _ = await _sync_request() + except httpx.HTTPStatusError as e: + raise RegistryError( + f"Failed to sync executor: HTTP {e.response.status_code}" + ) from e + except httpx.RequestError as e: + raise RegistryError( + f"Network error while syncing executor: {str(e)}" + ) from e + except Exception as e: + raise RegistryError( + f"Unexpected error while syncing executor: {str(e)}" + ) from e + """Registry management""" async def list_repositories(self) -> list[str]: diff --git a/tracecat/registry/constants.py b/tracecat/registry/constants.py index 25167d959..311598999 100644 --- a/tracecat/registry/constants.py +++ b/tracecat/registry/constants.py @@ -3,8 +3,8 @@ CUSTOM_REPOSITORY_ORIGIN = "custom" GITHUB_SSH_KEY_SECRET_NAME = "github-ssh-key" -REGISTRY_REPOS_PATH: str = "/repos" +REGISTRY_REPOS_PATH: str = "/registry/repos" """Base path for repository-related endpoints""" -REGISTRY_ACTIONS_PATH: str = "/actions" +REGISTRY_ACTIONS_PATH: str = "/registry/actions" """Base path for action-related endpoints""" diff --git a/tracecat/registry/executor.py b/tracecat/registry/executor.py index ffd7de38a..6eb4b8ec8 100644 --- a/tracecat/registry/executor.py +++ b/tracecat/registry/executor.py @@ -6,16 +6,21 @@ from __future__ import annotations import asyncio +import traceback from collections.abc import Iterator, Mapping from concurrent.futures import ProcessPoolExecutor from typing import Any, cast import uvloop +from fastapi import APIRouter, HTTPException, status +from pydantic import BaseModel from tracecat import config +from tracecat.auth.credentials import RoleACL from tracecat.auth.sandbox import AuthSandbox from tracecat.concurrency import GatheringTaskGroup from tracecat.contexts import ctx_logger, ctx_role, ctx_run +from tracecat.db.dependencies import AsyncDBSession from tracecat.db.engine import get_async_engine from tracecat.dsl.common import context_locator, create_default_dsl_context from tracecat.dsl.models import ( @@ -33,20 +38,132 @@ from tracecat.expressions.shared import ExprContext from tracecat.logger import logger from tracecat.parse import traverse_leaves -from tracecat.registry.actions.models import ArgsClsT, BoundRegistryAction +from tracecat.registry.actions.models import ( + ArgsClsT, + BoundRegistryAction, + RegistryActionErrorInfo, + RegistryActionValidate, + RegistryActionValidateResponse, +) from tracecat.registry.actions.service import RegistryActionsService +from tracecat.registry.repository import Repository from tracecat.secrets.common import apply_masks_object from tracecat.secrets.constants import DEFAULT_SECRETS_ENVIRONMENT from tracecat.secrets.secrets_manager import env_sandbox from tracecat.types.auth import Role -from tracecat.types.exceptions import TracecatException +from tracecat.types.exceptions import RegistryError, TracecatException +from tracecat.validation.service import validate_registry_action_args """All these methods are used in the registry executor, not on the worker""" -type ArgsT = Mapping[str, Any] +# Registry Action Controls +type ArgsT = Mapping[str, Any] _executor: ProcessPoolExecutor | None = None +router = APIRouter(tags=["executor"]) + + +class ExecutorSyncInput(BaseModel): + origin: str + + +@router.post("/sync") +async def sync_executor( + *, + role: Role = RoleACL( + allow_user=False, # XXX(authz): Users cannot sync the executor + allow_service=True, # Only services can sync the executor + require_workspace="no", + ), + input: ExecutorSyncInput, +) -> None: + """Sync the executor from the registry.""" + repo = Repository(origin=input.origin, role=role) + try: + await repo.load_from_origin() + except RegistryError as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e) + ) from e + + +@router.post("/run/{action_name}") +async def run_action( + *, + role: Role = RoleACL( + allow_user=False, # XXX(authz): Users cannot execute actions + allow_service=True, # Only services can execute actions + require_workspace="no", + ), + action_name: str, + action_input: RunActionInput, +) -> Any: + """Execute a registry action.""" + ref = action_input.task.ref + ctx_role.set(role) + act_logger = logger.bind(role=role, action_name=action_name, ref=ref) + ctx_logger.set(act_logger) + + act_logger.info("Starting action") + try: + return await run_action_in_pool(input=action_input) + except Exception as e: + # Get the traceback info + tb = traceback.extract_tb(e.__traceback__)[-1] # Get the last frame + error_detail = RegistryActionErrorInfo( + action_name=action_name, + type=e.__class__.__name__, + message=str(e), + filename=tb.filename, + function=tb.name, + lineno=tb.lineno, + ) + act_logger.error( + "Error running action", + action_name=action_name, + type=error_detail.type, + message=error_detail.message, + filename=error_detail.filename, + function=error_detail.function, + lineno=error_detail.lineno, + ) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=error_detail.model_dump(mode="json"), + ) from e + + +@router.post("/validate/{action_name}") +async def validate_action( + *, + role: Role = RoleACL( + allow_user=False, # XXX(authz): Users cannot validate actions + allow_service=True, # Only services can validate actions + require_workspace="no", + ), + session: AsyncDBSession, + action_name: str, + params: RegistryActionValidate, +) -> RegistryActionValidateResponse: + """Validate a registry action.""" + try: + result = await validate_registry_action_args( + session=session, action_name=action_name, args=params.args + ) + + if result.status == "error": + logger.warning( + "Error validating UDF args", message=result.msg, details=result.detail + ) + return RegistryActionValidateResponse.from_validation_result(result) + except KeyError as e: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Action {action_name!r} not found in registry", + ) from e + + # We want to be able to serve a looped action # Before we send out tasks to the executor we should inspect the size of the loop # and set the right chunk size for each worker diff --git a/tracecat/workflow/executions/service.py b/tracecat/workflow/executions/service.py index 2cbe025ad..6a71fbb75 100644 --- a/tracecat/workflow/executions/service.py +++ b/tracecat/workflow/executions/service.py @@ -19,6 +19,7 @@ WorkflowHandle, WorkflowHistoryEventFilterType, ) +from temporalio.service import RPCError from tracecat import config from tracecat.contexts import ctx_role @@ -435,6 +436,15 @@ async def _dispatch_workflow( except WorkflowFailureError as e: self.logger.error(str(e), role=self.role, wf_exec_id=wf_exec_id, e=e) raise e + except RPCError as e: + self.logger.error( + f"Temporal service RPC error occurred while executing the workflow: {e}", + role=self.role, + wf_exec_id=wf_exec_id, + e=e, + ) + raise e + except Exception as e: self.logger.exception( "Unexpected workflow error", role=self.role, wf_exec_id=wf_exec_id, e=e