[Rules migration] Retry failed translations (elastic#11383) (elastic#…

…204619) ## Summary [Internal link](elastic/security-team#10820) to the feature details These changes add a functionality which allows user to retry failed migration rules. ### Other tasks and fixes * Integrated `MigrationReadyPanel` and `MigrationProgressPanel` to show migration's `ready` and `running` states * Migration stats pooling issue caused by waiting while there are no pending migrations left. If any other operation triggers `startPooling` during the waiting it will be ignored and thus latest stats will never come back. > [!NOTE] > This feature needs `siemMigrationsEnabled` experimental flag enabled to work. ### Testing note 1. Make sure you have a SIEM migration with failed rules 2. Open that migration via `Security > Rules > SIEM Rules Migrations > {#MIGRATION_WITH_FAILED_RULES}` 3. You should see a `Reprocess rules (#)` button which triggers failed rules reprocessing ## Screen recording https://github.com/user-attachments/assets/d33dc4a0-1791-4869-aa8d-b0322b5f19c3 --------- Co-authored-by: kibanamachine <[email protected]>
crespocarlos · Jan 8, 2025 · ff00c38 · ff00c38
1 parent bd4df96
commit ff00c38
Show file tree

Hide file tree

Showing 28 changed files with 527 additions and 123 deletions.
diff --git a/x-pack/solutions/security/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/solutions/security/plugins/security_solution/common/api/quickstart_client.gen.ts
@@ -383,6 +383,9 @@ import type {
   InstallMigrationRulesResponse,
   InstallTranslatedMigrationRulesRequestParamsInput,
   InstallTranslatedMigrationRulesResponse,
+  RetryRuleMigrationRequestParamsInput,
+  RetryRuleMigrationRequestBodyInput,
+  RetryRuleMigrationResponse,
   StartRuleMigrationRequestParamsInput,
   StartRuleMigrationRequestBodyInput,
   StartRuleMigrationResponse,
@@ -2010,6 +2013,22 @@ detection engine rules.
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  /**
+   * Retries a SIEM rules migration using the migration id provided
+   */
+  async retryRuleMigration(props: RetryRuleMigrationProps) {
+    this.log.info(`${new Date().toISOString()} Calling API RetryRuleMigration`);
+    return this.kbnClient
+      .request<RetryRuleMigrationResponse>({
+        path: replaceParams('/internal/siem_migrations/rules/{migration_id}/retry', props.params),
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+        },
+        method: 'PUT',
+        body: props.body,
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   async riskEngineGetPrivileges() {
     this.log.info(`${new Date().toISOString()} Calling API RiskEngineGetPrivileges`);
     return this.kbnClient
@@ -2545,6 +2564,10 @@ export interface ReadRuleProps {
 export interface ResolveTimelineProps {
   query: ResolveTimelineRequestQueryInput;
 }
+export interface RetryRuleMigrationProps {
+  params: RetryRuleMigrationRequestParamsInput;
+  body: RetryRuleMigrationRequestBodyInput;
+}
 export interface RulePreviewProps {
   query: RulePreviewRequestQueryInput;
   body: RulePreviewRequestBodyInput;

diff --git a/...ty/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts b/...ty/plugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.gen.ts
@@ -214,6 +214,35 @@ export const InstallTranslatedMigrationRulesResponse = z.object({
   installed: z.boolean(),
 });
 
+export type RetryRuleMigrationRequestParams = z.infer<typeof RetryRuleMigrationRequestParams>;
+export const RetryRuleMigrationRequestParams = z.object({
+  migration_id: NonEmptyString,
+});
+export type RetryRuleMigrationRequestParamsInput = z.input<typeof RetryRuleMigrationRequestParams>;
+
+export type RetryRuleMigrationRequestBody = z.infer<typeof RetryRuleMigrationRequestBody>;
+export const RetryRuleMigrationRequestBody = z.object({
+  connector_id: ConnectorId,
+  langsmith_options: LangSmithOptions.optional(),
+  /**
+   * The indicator to retry only failed rules
+   */
+  failed: z.boolean().optional(),
+  /**
+   * The indicator to retry only not fully translated rules
+   */
+  not_fully_translated: z.boolean().optional(),
+});
+export type RetryRuleMigrationRequestBodyInput = z.input<typeof RetryRuleMigrationRequestBody>;
+
+export type RetryRuleMigrationResponse = z.infer<typeof RetryRuleMigrationResponse>;
+export const RetryRuleMigrationResponse = z.object({
+  /**
+   * Indicates the migration retry has been started. `false` means the migration does not need to be retried.
+   */
+  started: z.boolean(),
+});
+
 export type StartRuleMigrationRequestParams = z.infer<typeof StartRuleMigrationRequestParams>;
 export const StartRuleMigrationRequestParams = z.object({
   migration_id: NonEmptyString,

diff --git a/...ugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml b/...ugins/security_solution/common/siem_migrations/model/api/rules/rule_migration.schema.yaml
@@ -286,6 +286,57 @@ paths:
         204:
           description: Indicates the migration id was not found.
 
+  /internal/siem_migrations/rules/{migration_id}/retry:
+    put:
+      summary: Retries a rule migration
+      operationId: RetryRuleMigration
+      x-codegen-enabled: true
+      x-internal: true
+      description: Retries a SIEM rules migration using the migration id provided
+      tags:
+        - SIEM Rule Migrations
+      parameters:
+        - name: migration_id
+          in: path
+          required: true
+          schema:
+            description: The migration id to retry
+            $ref: '../../../../../common/api/model/primitives.schema.yaml#/components/schemas/NonEmptyString'
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required:
+                - connector_id
+              properties:
+                connector_id:
+                  $ref: '../../common.schema.yaml#/components/schemas/ConnectorId'
+                langsmith_options:
+                  $ref: '../../common.schema.yaml#/components/schemas/LangSmithOptions'
+                failed:
+                  type: boolean
+                  description: The indicator to retry only failed rules
+                not_fully_translated:
+                  type: boolean
+                  description: The indicator to retry only not fully translated rules
+      responses:
+        200:
+          description: Indicates the migration retry request has been processed successfully.
+          content:
+            application/json:
+              schema:
+                type: object
+                required:
+                  - started
+                properties:
+                  started:
+                    type: boolean
+                    description: Indicates the migration retry has been started. `false` means the migration does not need to be retried.
+        204:
+          description: Indicates the migration id was not found.
+
   /internal/siem_migrations/rules/{migration_id}/stats:
     get:
       summary: Gets a rule migration task stats

diff --git a/...ck/solutions/security/plugins/security_solution/public/siem_migrations/rules/api/index.ts b/...ck/solutions/security/plugins/security_solution/public/siem_migrations/rules/api/index.ts
@@ -23,6 +23,7 @@ import {
   SIEM_RULE_MIGRATION_RESOURCES_MISSING_PATH,
   SIEM_RULE_MIGRATION_RESOURCES_PATH,
   SIEM_RULE_MIGRATIONS_PREBUILT_RULES_PATH,
+  SIEM_RULE_MIGRATION_RETRY_PATH,
 } from '../../../../common/siem_migrations/constants';
 import type {
   CreateRuleMigrationRequestBody,
@@ -39,6 +40,9 @@ import type {
   UpsertRuleMigrationResourcesResponse,
   GetRuleMigrationPrebuiltRulesResponse,
   UpdateRuleMigrationResponse,
+  RetryRuleMigrationRequestBody,
+  StartRuleMigrationResponse,
+  RetryRuleMigrationResponse,
 } from '../../../../common/siem_migrations/model/api/rules/rule_migration.gen';
 
 export interface GetRuleMigrationStatsParams {
@@ -146,17 +150,54 @@ export const startRuleMigration = async ({
   connectorId,
   langSmithOptions,
   signal,
-}: StartRuleMigrationParams): Promise<GetAllStatsRuleMigrationResponse> => {
+}: StartRuleMigrationParams): Promise<StartRuleMigrationResponse> => {
   const body: StartRuleMigrationRequestBody = { connector_id: connectorId };
   if (langSmithOptions) {
     body.langsmith_options = langSmithOptions;
   }
-  return KibanaServices.get().http.put<GetAllStatsRuleMigrationResponse>(
+  return KibanaServices.get().http.put<StartRuleMigrationResponse>(
     replaceParams(SIEM_RULE_MIGRATION_START_PATH, { migration_id: migrationId }),
     { body: JSON.stringify(body), version: '1', signal }
   );
 };
 
+export interface RetryRuleMigrationParams {
+  /** `id` of the migration to reprocess rules for */
+  migrationId: string;
+  /** The connector id to use for the reprocessing */
+  connectorId: string;
+  /** Optional LangSmithOptions to use for the for the reprocessing */
+  langSmithOptions?: LangSmithOptions;
+  /** Optional indicator to retry only failed rules */
+  failed?: boolean;
+  /** Optional indicator to retry only not fully translated rules */
+  notFullyTranslated?: boolean;
+  /** Optional AbortSignal for cancelling request */
+  signal?: AbortSignal;
+}
+/** Starts a reprocessing of migration rules in a specific migration. */
+export const retryRuleMigration = async ({
+  migrationId,
+  connectorId,
+  langSmithOptions,
+  failed,
+  notFullyTranslated,
+  signal,
+}: RetryRuleMigrationParams): Promise<RetryRuleMigrationResponse> => {
+  const body: RetryRuleMigrationRequestBody = {
+    connector_id: connectorId,
+    failed,
+    not_fully_translated: notFullyTranslated,
+  };
+  if (langSmithOptions) {
+    body.langsmith_options = langSmithOptions;
+  }
+  return KibanaServices.get().http.put<RetryRuleMigrationResponse>(
+    replaceParams(SIEM_RULE_MIGRATION_RETRY_PATH, { migration_id: migrationId }),
+    { body: JSON.stringify(body), version: '1', signal }
+  );
+};
+
 export interface GetRuleMigrationParams {
   /** `id` of the migration to get rules documents for */
   migrationId: string;

diff --git a/...ns/security_solution/public/siem_migrations/rules/components/rules_table/bulk_actions.tsx b/...ns/security_solution/public/siem_migrations/rules/components/rules_table/bulk_actions.tsx
@@ -6,21 +6,17 @@
  */
 
 import React from 'react';
-import {
-  EuiButton,
-  EuiButtonEmpty,
-  EuiFlexGroup,
-  EuiFlexItem,
-  EuiLoadingSpinner,
-} from '@elastic/eui';
+import { EuiButton, EuiButtonEmpty, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import * as i18n from './translations';
 
 export interface BulkActionsProps {
   isTableLoading: boolean;
+  numberOfFailedRules: number;
   numberOfTranslatedRules: number;
   numberOfSelectedRules: number;
   installTranslatedRule?: () => void;
   installSelectedRule?: () => void;
+  reprocessFailedRules?: () => void;
 }
 
 /**
@@ -29,43 +25,60 @@ export interface BulkActionsProps {
 export const BulkActions: React.FC<BulkActionsProps> = React.memo(
   ({
     isTableLoading,
+    numberOfFailedRules,
     numberOfTranslatedRules,
     numberOfSelectedRules,
     installTranslatedRule,
     installSelectedRule,
+    reprocessFailedRules,
   }) => {
     const disableInstallTranslatedRulesButton = isTableLoading || !numberOfTranslatedRules;
-    const showInstallSelectedRulesButton = isTableLoading || numberOfSelectedRules > 0;
+    const showInstallSelectedRulesButton = numberOfSelectedRules > 0;
+    const showRetryFailedRulesButton = numberOfFailedRules > 0;
     return (
       <EuiFlexGroup alignItems="center" gutterSize="s" responsive={false} wrap={true}>
-        {showInstallSelectedRulesButton ? (
+        {showInstallSelectedRulesButton && (
           <EuiFlexItem grow={false}>
             <EuiButtonEmpty
               iconType="plusInCircle"
               color={'primary'}
-              onClick={installSelectedRule}
+              onClick={() => installSelectedRule?.()}
               disabled={isTableLoading}
+              isLoading={isTableLoading}
               data-test-subj="installSelectedRulesButton"
               aria-label={i18n.INSTALL_SELECTED_ARIA_LABEL}
             >
               {i18n.INSTALL_SELECTED_RULES(numberOfSelectedRules)}
-              {isTableLoading && <EuiLoadingSpinner size="s" />}
             </EuiButtonEmpty>
           </EuiFlexItem>
-        ) : null}
+        )}
+        {showRetryFailedRulesButton && (
+          <EuiFlexItem grow={false}>
+            <EuiButton
+              iconType="refresh"
+              color={'warning'}
+              onClick={() => reprocessFailedRules?.()}
+              disabled={isTableLoading}
+              isLoading={isTableLoading}
+              data-test-subj="reprocessFailedRulesButton"
+              aria-label={i18n.REPROCESS_FAILED_ARIA_LABEL}
+            >
+              {i18n.REPROCESS_FAILED_RULES(numberOfFailedRules)}
+            </EuiButton>
+          </EuiFlexItem>
+        )}
         <EuiFlexItem grow={false}>
           <EuiButton
-            fill
             iconType="plusInCircle"
-            data-test-subj="installTranslatedRulesButton"
-            onClick={installTranslatedRule}
+            onClick={() => installTranslatedRule?.()}
             disabled={disableInstallTranslatedRulesButton}
+            isLoading={isTableLoading}
+            data-test-subj="installTranslatedRulesButton"
             aria-label={i18n.INSTALL_TRANSLATED_ARIA_LABEL}
           >
             {numberOfTranslatedRules > 0
               ? i18n.INSTALL_TRANSLATED_RULES(numberOfTranslatedRules)
               : i18n.INSTALL_TRANSLATED_RULES_EMPTY_STATE}
-            {isTableLoading && <EuiLoadingSpinner size="s" />}
           </EuiButton>
         </EuiFlexItem>
       </EuiFlexGroup>

diff --git a/...y/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx b/...y/plugins/security_solution/public/siem_migrations/rules/components/rules_table/index.tsx
@@ -33,6 +33,7 @@ import { BulkActions } from './bulk_actions';
 import { SearchField } from './search_field';
 import { RuleTranslationResult } from '../../../../../common/siem_migrations/constants';
 import * as i18n from './translations';
+import { useRetryRuleMigration } from '../../service/hooks/use_retry_rules';
 
 const DEFAULT_PAGE_SIZE = 10;
 const DEFAULT_SORT_FIELD = 'translation_result';
@@ -43,13 +44,18 @@ export interface MigrationRulesTableProps {
    * Selected rule migration id
    */
   migrationId: string;
+
+  /**
+   * Re-fetches latest rule migration data
+   */
+  refetchData?: () => void;
 }
 
 /**
  * Table Component for displaying SIEM rules migrations
  */
 export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.memo(
-  ({ migrationId }) => {
+  ({ migrationId, refetchData }) => {
     const { addError } = useAppToasts();
 
     const [pageIndex, setPageIndex] = useState(0);
@@ -132,6 +138,7 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
     const { mutateAsync: installMigrationRules } = useInstallMigrationRules(migrationId);
     const { mutateAsync: installTranslatedMigrationRules } =
       useInstallTranslatedMigrationRules(migrationId);
+    const { retryRuleMigration, isLoading: isRetryLoading } = useRetryRuleMigration(refetchData);
 
     const [isTableLoading, setTableLoading] = useState(false);
     const installSingleRule = useCallback(
@@ -180,7 +187,12 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
       [addError, installTranslatedMigrationRules]
     );
 
-    const isLoading = isStatsLoading || isPrebuiltRulesLoading || isDataLoading || isTableLoading;
+    const reprocessFailedRules = useCallback(async () => {
+      retryRuleMigration(migrationId, { failed: true });
+    }, [migrationId, retryRuleMigration]);
+
+    const isLoading =
+      isStatsLoading || isPrebuiltRulesLoading || isDataLoading || isTableLoading || isRetryLoading;
 
     const ruleActionsFactory = useCallback(
       (ruleMigration: RuleMigration, closeRulePreview: () => void) => {
@@ -268,10 +280,12 @@ export const MigrationRulesTable: React.FC<MigrationRulesTableProps> = React.mem
                   <EuiFlexItem grow={false}>
                     <BulkActions
                       isTableLoading={isLoading}
-                      numberOfTranslatedRules={translationStats?.rules.success.installable ?? 0}
+                      numberOfFailedRules={translationStats.rules.failed}
+                      numberOfTranslatedRules={translationStats.rules.success.installable}
                       numberOfSelectedRules={selectedRuleMigrations.length}
                       installTranslatedRule={installTranslatedRules}
                       installSelectedRule={installSelectedRule}
+                      reprocessFailedRules={reprocessFailedRules}
                     />
                   </EuiFlexItem>
                 </EuiFlexGroup>

diff --git a/...ins/security_solution/public/siem_migrations/rules/components/rules_table/translations.ts b/...ins/security_solution/public/siem_migrations/rules/components/rules_table/translations.ts
@@ -49,6 +49,13 @@ export const INSTALL_SELECTED_RULES = (numberOfSelectedRules: number) => {
   });
 };
 
+export const REPROCESS_FAILED_RULES = (numberOfFailedRules: number) => {
+  return i18n.translate('xpack.securitySolution.siemMigrations.rules.table.reprocessFailedRules', {
+    defaultMessage: 'Reprocess rules ({numberOfFailedRules})',
+    values: { numberOfFailedRules },
+  });
+};
+
 export const INSTALL_TRANSLATED_RULES_EMPTY_STATE = i18n.translate(
   'xpack.securitySolution.siemMigrations.rules.table.installTranslatedRulesEmptyState',
   {
@@ -81,6 +88,13 @@ export const INSTALL_TRANSLATED_ARIA_LABEL = i18n.translate(
   }
 );
 
+export const REPROCESS_FAILED_ARIA_LABEL = i18n.translate(
+  'xpack.securitySolution.siemMigrations.rules.table.reprocessFailedRulesButtonAriaLabel',
+  {
+    defaultMessage: 'Reprocess failed rules',
+  }
+);
+
 export const ALREADY_TRANSLATED_RULE_TOOLTIP = i18n.translate(
   'xpack.securitySolution.siemMigrations.rules.table.alreadyTranslatedTooltip',
   {