Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Rules migration] Retry failed translations (#11383) #204619

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,7 @@ import type { DataView } from '@kbn/data-views-plugin/public';
import { Warnings } from '@kbn/charts-plugin/public';
import { hasUnsupportedDownsampledAggregationFailure } from '@kbn/search-response-warnings';
import { Adapters } from '@kbn/inspector-plugin/public';
import {
Embeddable,
EmbeddableInput,
EmbeddableOutput,
} from '@kbn/embeddable-plugin/public';
import { Embeddable, EmbeddableInput, EmbeddableOutput } from '@kbn/embeddable-plugin/public';
semd marked this conversation as resolved.
Show resolved Hide resolved
import { SavedObjectEmbeddableInput } from '@kbn/embeddable-plugin/common';
import {
ExpressionAstExpression,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,9 @@ import type {
InstallMigrationRulesResponse,
InstallTranslatedMigrationRulesRequestParamsInput,
InstallTranslatedMigrationRulesResponse,
RetryRuleMigrationRequestParamsInput,
RetryRuleMigrationRequestBodyInput,
RetryRuleMigrationResponse,
StartRuleMigrationRequestParamsInput,
StartRuleMigrationRequestBodyInput,
StartRuleMigrationResponse,
Expand Down Expand Up @@ -2010,6 +2013,22 @@ detection engine rules.
})
.catch(catchAxiosErrorFormatAndThrow);
}
/**
* Retries a SIEM rules migration using the migration id provided
*/
async retryRuleMigration(props: RetryRuleMigrationProps) {
this.log.info(`${new Date().toISOString()} Calling API RetryRuleMigration`);
return this.kbnClient
.request<RetryRuleMigrationResponse>({
path: replaceParams('/internal/siem_migrations/rules/{migration_id}/retry', props.params),
headers: {
[ELASTIC_HTTP_VERSION_HEADER]: '1',
},
method: 'PUT',
body: props.body,
})
.catch(catchAxiosErrorFormatAndThrow);
}
async riskEngineGetPrivileges() {
this.log.info(`${new Date().toISOString()} Calling API RiskEngineGetPrivileges`);
return this.kbnClient
Expand Down Expand Up @@ -2545,6 +2564,10 @@ export interface ReadRuleProps {
export interface ResolveTimelineProps {
query: ResolveTimelineRequestQueryInput;
}
export interface RetryRuleMigrationProps {
params: RetryRuleMigrationRequestParamsInput;
body: RetryRuleMigrationRequestBodyInput;
}
export interface RulePreviewProps {
query: RulePreviewRequestQueryInput;
body: RulePreviewRequestBodyInput;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,35 @@ export const InstallTranslatedMigrationRulesResponse = z.object({
installed: z.boolean(),
});

export type RetryRuleMigrationRequestParams = z.infer<typeof RetryRuleMigrationRequestParams>;
export const RetryRuleMigrationRequestParams = z.object({
migration_id: NonEmptyString,
});
export type RetryRuleMigrationRequestParamsInput = z.input<typeof RetryRuleMigrationRequestParams>;

export type RetryRuleMigrationRequestBody = z.infer<typeof RetryRuleMigrationRequestBody>;
export const RetryRuleMigrationRequestBody = z.object({
connector_id: ConnectorId,
langsmith_options: LangSmithOptions.optional(),
/**
* The indicator to retry only failed rules
*/
failed: z.boolean().optional(),
/**
* The indicator to retry only not fully translated rules
*/
not_fully_translated: z.boolean().optional(),
});
export type RetryRuleMigrationRequestBodyInput = z.input<typeof RetryRuleMigrationRequestBody>;

export type RetryRuleMigrationResponse = z.infer<typeof RetryRuleMigrationResponse>;
export const RetryRuleMigrationResponse = z.object({
/**
* Indicates the migration retry has been started. `false` means the migration does not need to be retried.
*/
started: z.boolean(),
});

export type StartRuleMigrationRequestParams = z.infer<typeof StartRuleMigrationRequestParams>;
export const StartRuleMigrationRequestParams = z.object({
migration_id: NonEmptyString,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,57 @@ paths:
204:
description: Indicates the migration id was not found.

/internal/siem_migrations/rules/{migration_id}/retry:
put:
summary: Retries a rule migration
operationId: RetryRuleMigration
x-codegen-enabled: true
x-internal: true
description: Retries a SIEM rules migration using the migration id provided
tags:
- SIEM Rule Migrations
parameters:
- name: migration_id
in: path
required: true
schema:
description: The migration id to retry
$ref: '../../../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString'
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- connector_id
properties:
connector_id:
$ref: '../../common.schema.yaml#/components/schemas/ConnectorId'
langsmith_options:
$ref: '../../common.schema.yaml#/components/schemas/LangSmithOptions'
failed:
type: boolean
description: The indicator to retry only failed rules
not_fully_translated:
type: boolean
description: The indicator to retry only not fully translated rules
responses:
200:
description: Indicates the migration retry request has been processed successfully.
content:
application/json:
schema:
type: object
required:
- started
properties:
started:
type: boolean
description: Indicates the migration retry has been started. `false` means the migration does not need to be retried.
204:
description: Indicates the migration id was not found.

/internal/siem_migrations/rules/{migration_id}/stats:
get:
summary: Gets a rule migration task stats
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import {
SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH,
SIEM_RULE_MIGRATION_RESOURCES_PATH,
SIEM_RULE_MIGRATIONS_PREBUILT_RULES_PATH,
SIEM_RULE_MIGRATION_RETRY_PATH,
} from '../../../../common/siem_migrations/constants';
import type {
CreateRuleMigrationRequestBody,
Expand All @@ -39,6 +40,9 @@ import type {
UpsertRuleMigrationResourcesResponse,
GetRuleMigrationPrebuiltRulesResponse,
UpdateRuleMigrationResponse,
RetryRuleMigrationRequestBody,
StartRuleMigrationResponse,
RetryRuleMigrationResponse,
} from '../../../../common/siem_migrations/model/api/rules/rule_migration.gen';

export interface GetRuleMigrationStatsParams {
Expand Down Expand Up @@ -146,17 +150,54 @@ export const startRuleMigration = async ({
connectorId,
langSmithOptions,
signal,
}: StartRuleMigrationParams): Promise<GetAllStatsRuleMigrationResponse> => {
}: StartRuleMigrationParams): Promise<StartRuleMigrationResponse> => {
const body: StartRuleMigrationRequestBody = { connector_id: connectorId };
if (langSmithOptions) {
body.langsmith_options = langSmithOptions;
}
return KibanaServices.get().http.put<GetAllStatsRuleMigrationResponse>(
return KibanaServices.get().http.put<StartRuleMigrationResponse>(
replaceParams(SIEM_RULE_MIGRATION_START_PATH, { migration_id: migrationId }),
{ body: JSON.stringify(body), version: '1', signal }
);
};

export interface RetryRuleMigrationParams {
/** `id` of the migration to reprocess rules for */
migrationId: string;
/** The connector id to use for the reprocessing */
connectorId: string;
/** Optional LangSmithOptions to use for the for the reprocessing */
langSmithOptions?: LangSmithOptions;
/** Optional indicator to retry only failed rules */
failed?: boolean;
/** Optional indicator to retry only not fully translated rules */
notFullyTranslated?: boolean;
/** Optional AbortSignal for cancelling request */
signal?: AbortSignal;
}
/** Starts a reprocessing of migration rules in a specific migration. */
export const retryRuleMigration = async ({
migrationId,
connectorId,
langSmithOptions,
failed,
notFullyTranslated,
signal,
}: RetryRuleMigrationParams): Promise<RetryRuleMigrationResponse> => {
const body: RetryRuleMigrationRequestBody = {
connector_id: connectorId,
failed,
not_fully_translated: notFullyTranslated,
};
if (langSmithOptions) {
body.langsmith_options = langSmithOptions;
}
return KibanaServices.get().http.put<RetryRuleMigrationResponse>(
replaceParams(SIEM_RULE_MIGRATION_RETRY_PATH, { migration_id: migrationId }),
{ body: JSON.stringify(body), version: '1', signal }
);
};

export interface GetRuleMigrationParams {
/** `id` of the migration to get rules documents for */
migrationId: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,17 @@
*/

import React from 'react';
import {
EuiButton,
EuiButtonEmpty,
EuiFlexGroup,
EuiFlexItem,
EuiLoadingSpinner,
} from '@elastic/eui';
import { EuiButton, EuiButtonEmpty, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
import * as i18n from './translations';

export interface BulkActionsProps {
isTableLoading: boolean;
numberOfFailedRules: number;
numberOfTranslatedRules: number;
numberOfSelectedRules: number;
installTranslatedRule?: () => void;
installSelectedRule?: () => void;
reprocessFailedRules?: () => void;
}

/**
Expand All @@ -29,43 +25,60 @@ export interface BulkActionsProps {
export const BulkActions: React.FC<BulkActionsProps> = React.memo(
({
isTableLoading,
numberOfFailedRules,
numberOfTranslatedRules,
numberOfSelectedRules,
installTranslatedRule,
installSelectedRule,
reprocessFailedRules,
}) => {
const disableInstallTranslatedRulesButton = isTableLoading || !numberOfTranslatedRules;
const showInstallSelectedRulesButton = isTableLoading || numberOfSelectedRules > 0;
const showInstallSelectedRulesButton = numberOfSelectedRules > 0;
const showRetryFailedRulesButton = numberOfFailedRules > 0;
return (
<EuiFlexGroup alignItems="center" gutterSize="s" responsive={false} wrap={true}>
{showInstallSelectedRulesButton ? (
{showInstallSelectedRulesButton && (
<EuiFlexItem grow={false}>
<EuiButtonEmpty
iconType="plusInCircle"
color={'primary'}
onClick={installSelectedRule}
onClick={() => installSelectedRule?.()}
disabled={isTableLoading}
isLoading={isTableLoading}
data-test-subj="installSelectedRulesButton"
aria-label={i18n.INSTALL_SELECTED_ARIA_LABEL}
>
{i18n.INSTALL_SELECTED_RULES(numberOfSelectedRules)}
{isTableLoading && <EuiLoadingSpinner size="s" />}
</EuiButtonEmpty>
</EuiFlexItem>
) : null}
)}
{showRetryFailedRulesButton && (
<EuiFlexItem grow={false}>
<EuiButton
iconType="refresh"
color={'warning'}
onClick={() => reprocessFailedRules?.()}
disabled={isTableLoading}
isLoading={isTableLoading}
data-test-subj="reprocessFailedRulesButton"
aria-label={i18n.REPROCESS_FAILED_ARIA_LABEL}
>
{i18n.REPROCESS_FAILED_RULES(numberOfFailedRules)}
</EuiButton>
</EuiFlexItem>
)}
<EuiFlexItem grow={false}>
<EuiButton
fill
iconType="plusInCircle"
data-test-subj="installTranslatedRulesButton"
onClick={installTranslatedRule}
onClick={() => installTranslatedRule?.()}
disabled={disableInstallTranslatedRulesButton}
isLoading={isTableLoading}
data-test-subj="installTranslatedRulesButton"
aria-label={i18n.INSTALL_TRANSLATED_ARIA_LABEL}
>
{numberOfTranslatedRules > 0
? i18n.INSTALL_TRANSLATED_RULES(numberOfTranslatedRules)
: i18n.INSTALL_TRANSLATED_RULES_EMPTY_STATE}
{isTableLoading && <EuiLoadingSpinner size="s" />}
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import { BulkActions } from './bulk_actions';
import { SearchField } from './search_field';
import { RuleTranslationResult } from '../../../../../common/siem_migrations/constants';
import * as i18n from './translations';
import { useRetryRuleMigration } from '../../service/hooks/use_retry_rules';

const DEFAULT_PAGE_SIZE = 10;
const DEFAULT_SORT_FIELD = 'translation_result';
Expand All @@ -43,13 +44,18 @@ export interface MigrationRulesTableProps {
* Selected rule migration id
*/
migrationId: string;

/**
* Re-fetches latest rule migration data
*/
refetchData?: () => void;
}

/**
* Table Component for displaying SIEM rules migrations
*/
export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.memo(
({ migrationId }) => {
({ migrationId, refetchData }) => {
const { addError } = useAppToasts();

const [pageIndex, setPageIndex] = useState(0);
Expand Down Expand Up @@ -132,6 +138,7 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
const { mutateAsync: installMigrationRules } = useInstallMigrationRules(migrationId);
const { mutateAsync: installTranslatedMigrationRules } =
useInstallTranslatedMigrationRules(migrationId);
const { retryRuleMigration, isLoading: isRetryLoading } = useRetryRuleMigration(refetchData);

const [isTableLoading, setTableLoading] = useState(false);
const installSingleRule = useCallback(
Expand Down Expand Up @@ -180,7 +187,12 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
[addError, installTranslatedMigrationRules]
);

const isLoading = isStatsLoading || isPrebuiltRulesLoading || isDataLoading || isTableLoading;
const reprocessFailedRules = useCallback(async () => {
retryRuleMigration(migrationId, { failed: true });
}, [migrationId, retryRuleMigration]);

const isLoading =
isStatsLoading || isPrebuiltRulesLoading || isDataLoading || isTableLoading || isRetryLoading;

const ruleActionsFactory = useCallback(
(ruleMigration: RuleMigration, closeRulePreview: () => void) => {
Expand Down Expand Up @@ -268,10 +280,12 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
<EuiFlexItem grow={false}>
<BulkActions
isTableLoading={isLoading}
numberOfTranslatedRules={translationStats?.rules.success.installable ?? 0}
numberOfFailedRules={translationStats.rules.failed}
numberOfTranslatedRules={translationStats.rules.success.installable}
numberOfSelectedRules={selectedRuleMigrations.length}
installTranslatedRule={installTranslatedRules}
installSelectedRule={installSelectedRule}
reprocessFailedRules={reprocessFailedRules}
/>
</EuiFlexItem>
</EuiFlexGroup>
Expand Down
Loading