[Security Solution][Detection Engine] add deprecation warning for non…

…-migrated signals (elastic#204247)

## Summary

- addresses partly elastic/security-team#10878
 - shows deprecation warning if siem index was not migrated


### How to test

#### How to create legacy siem index?

run script that used for FTR tests

```bash
node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index

node scripts/es_archiver --kibana-url=http://elastic:changeme@localhost:5601 --es-url=http://elastic:changeme@localhost:9200 load x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space
```
These would create legacy siem indices. But be aware, it might break
Kibana .alerts indices creation. But sufficient for testing

Visit also detection rules page, to ensure alerts index created.
Otherwise,
https://www.elastic.co/guide/en/security/current/signals-migration-api.html#migration-1
API might not show these indices outdated

#### How to test deprecated feature?
1. Observe warning feature deprecation on Kibana Upgrade page, if you
set up legacy siem signals

<details>
<summary> Kibana Upgrade feature deprecation flyout </summary>

<img width="2540" alt="Screenshot 2024-12-17 at 16 59 04"
src="https://github.com/user-attachments/assets/c6aa420f-af69-4545-8400-6a6513f613a9"
/>



 </details>

#### Test outdated indices created in 7.x

1. Create cloud env of 7.x version
2. Create rule, generate alerts for .siem-signals
3. Create cloud env of 8.18 from existing 7.x snapshot (from previous
steps)
4. Connect local Kibana to 8.18 from mirror branch of this
one(elastic#204621)
5. Add to Kibana dev config following options to enable Upgrade
assistant(UA) showing outdated indices
    ```yml
    xpack.upgrade_assistant.featureSet:
      mlSnapshots: true
      migrateDataStreams: true
      migrateSystemIndices: true
      reindexCorrectiveActions: true
    ```  
6. Go to Detection rules page, ensure rule is running and new .alerts
index has been created (visiting rules table page should be enough)
7. Open UA, ensure Kibana deprecations show signals are not migrated
8. Open UA, check Elasticsearch deprecations
9. Find outdated siem-signals index
10. Migrate it
11. Check Kibana deprecations still  signals are not migrated
12. Migrate signals using
https://www.elastic.co/guide/en/security/current/signals-migration-api.html
API
13. Ensure Kibana deprecations does not show that space as not migrated

Demo video of migration .siem-signal from another-3 Kibana space


https://github.com/user-attachments/assets/d2729482-d2c8-4a23-a780-ad19d4f52c73

x-pack/solutions/security/plugins/security_solution/server/deprecations/index.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type { CoreSetup, Logger } from '@kbn/core/server';
+import type { ConfigType } from '../config';
+
+import { getSignalsMigrationDeprecationsInfo } from './signals_migration';
+
+export const registerDeprecations = ({
+  core,
+  config,
+  logger,
+}: {
+  core: CoreSetup;
+  config: ConfigType;
+  logger: Logger;
+}) => {
+  core.deprecations.registerDeprecations({
+    getDeprecations: async (ctx) => {
+      return [...(await getSignalsMigrationDeprecationsInfo(ctx, config, logger, core.docLinks))];
+    },
+  });
+};

x-pack/solutions/security/plugins/security_solution/server/deprecations/signals_migration.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  DeprecationsDetails,
+  GetDeprecationsContext,
+  Logger,
+  DocLinksServiceSetup,
+} from '@kbn/core/server';
+
+import { i18n } from '@kbn/i18n';
+import { DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL } from '../../common/constants';
+import type { ConfigType } from '../config';
+
+import { getNonMigratedSignalsInfo } from '../lib/detection_engine/migrations/get_non_migrated_signals_info';
+
+const constructMigrationApiCall = (space: string, range: string) =>
+  `GET <kibana host>:<port>${
+    space === 'default' ? '' : `/s/${space}`
+  }${DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL}?from=${range}`;
+
+export const getSignalsMigrationDeprecationsInfo = async (
+  ctx: GetDeprecationsContext,
+  config: ConfigType,
+  logger: Logger,
+  docLinks: DocLinksServiceSetup
+): Promise<DeprecationsDetails[]> => {
+  const esClient = ctx.esClient.asInternalUser;
+  const { isMigrationRequired, spaces } = await getNonMigratedSignalsInfo({
+    esClient,
+    signalsIndex: config.signalsIndex,
+    logger,
+  });
+  // Deprecation API requires time range to be part of request (https://www.elastic.co/guide/en/security/current/signals-migration-api.html#migration-1)
+  // Return the earliest date, so it would capture the oldest possible signals
+  const fromRange = new Date(0).toISOString();
+
+  if (isMigrationRequired) {
+    return [
+      {
+        deprecationType: 'feature',
+        title: i18n.translate('xpack.securitySolution.deprecations.signalsMigrationTitle', {
+          defaultMessage: 'Found not migrated detection alerts',
+        }),
+        level: 'warning',
+        message: i18n.translate('xpack.securitySolution.deprecations.signalsMigrationMessage', {
+          defaultMessage: `After upgrading Kibana, the latest Elastic Security features will be available for any newly generated detection alerts. However, in order to enable new features for existing detection alerts, migration may be necessary.`,
+        }),
+        documentationUrl: docLinks.links.securitySolution.signalsMigrationApi,
+        correctiveActions: {
+          manualSteps: [
+            i18n.translate(
+              'xpack.securitySolution.deprecations.migrateIndexIlmPolicy.signalsMigrationManualStepOne',
+              {
+                defaultMessage: `Visit "Learn more" link for instructions how to migrate detection alerts. Migrate indices for each space.`,
+              }
+            ),
+            i18n.translate(
+              'xpack.securitySolution.deprecations.migrateIndexIlmPolicy.signalsMigrationManualStepTwo',
+              {
+                defaultMessage: 'Spaces with at least one non-migrated signals index: {spaces}.',
+                values: {
+                  spaces: spaces.join(', '),
+                },
+              }
+            ),
+            i18n.translate(
+              'xpack.securitySolution.deprecations.migrateIndexIlmPolicy.signalsMigrationManualStepFour',
+              {
+                defaultMessage: 'Example of migration API calls:',
+              }
+            ),
+            ...spaces.map((space) => constructMigrationApiCall(space, fromRange)),
+          ],
+        },
+      },
+    ];
+  }
+
+  return [];
+};

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_index.ts

@@ -42,6 +42,11 @@ export const createMigrationIndex = async ({
           },
         },
       },
+      mappings: {
+        _meta: {
+          version,
+        },
+      },
     },
   });
 

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.test.ts

@@ -7,7 +7,10 @@
 import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
 import { loggerMock } from '@kbn/logging-mocks';
 
-import { getNonMigratedSignalsInfo } from './get_non_migrated_signals_info';
+import {
+  getNonMigratedSignalsInfo,
+  checkIfMigratedIndexOutdated,
+} from './get_non_migrated_signals_info';
 import { getIndexVersionsByIndex } from './get_index_versions_by_index';
 import { getSignalVersionsByIndex } from './get_signal_versions_by_index';
 import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
@@ -132,6 +135,39 @@ describe('getNonMigratedSignalsInfo', () => {
       spaces: ['default'],
     });
   });
+  it('return empty result for migrated in v8 index', async () => {
+    getIndexAliasPerSpaceMock.mockReturnValue({
+      '.reindexed-v8-siem-signals-another-1-000001': {
+        alias: '.siem-signals-another-1',
+        indexName: '.reindexed-v8-siem-signals-another-1-000001',
+        space: 'another-1-000001',
+      },
+      '.siem-signals-another-1-000002': {
+        alias: '.siem-signals-another-1',
+        indexName: '.siem-signals-another-1-000002',
+        space: 'another-1',
+      },
+    });
+
+    getIndexVersionsByIndexMock.mockReturnValue({
+      '.reindexed-v8-siem-signals-another-1-000001': 57,
+      '.siem-signals-another-1-000002': TEMPLATE_VERSION,
+      '.reindexed-v8-siem-signals-another-1-000001-r000077': TEMPLATE_VERSION, // outdated .reindexed-v8-siem-signals-another-1-000001 is already migrated
+    });
+    getSignalVersionsByIndexMock.mockReturnValue({});
+
+    const result = await getNonMigratedSignalsInfo({
+      esClient,
+      signalsIndex: 'siem-signals',
+      logger,
+    });
+
+    expect(result).toEqual({
+      indices: [],
+      isMigrationRequired: false,
+      spaces: [],
+    });
+  });
   it('returns results for outdated signals in index', async () => {
     getIndexVersionsByIndexMock.mockReturnValue({
       '.siem-signals-another-1-legacy': TEMPLATE_VERSION,
@@ -175,3 +211,49 @@ describe('getNonMigratedSignalsInfo', () => {
     });
   });
 });
+
+describe('checkIfMigratedIndexOutdated', () => {
+  const indexVersionsByIndex = {
+    '.siem-signals-default-000001': 57,
+    '.siem-signals-another-6-000001': 57,
+    '.siem-signals-default-000002': 77,
+    '.siem-signals-another-5-000001': 57,
+    '.reindexed-v8-siem-signals-another-1-000001': 57,
+    '.siem-signals-another-7-000001': 57,
+    '.reindexed-v8-siem-signals-another-2-000001': 57,
+    '.siem-signals-another-3-000001': 57,
+    '.reindexed-v8-siem-signals-another-4-000001': 57,
+    '.siem-signals-another-3-000002': 77,
+    '.siem-signals-another-9-000001': 57,
+    '.siem-signals-another-8-000001': 57,
+    '.siem-signals-another-2-000002': 77,
+    '.siem-signals-another-10-000001': 57,
+    '.siem-signals-another-1-000002': 77,
+    '.siem-signals-another-2-000001-r000077': 77,
+    '.reindexed-v8-siem-signals-another-1-000001-r000077': 77,
+  };
+
+  const migratedIndices = [
+    '.reindexed-v8-siem-signals-another-1-000001',
+    '.reindexed-v8-siem-signals-another-2-000001',
+    '.reindexed-v8-siem-signals-another-1-000001-r000077',
+  ];
+
+  migratedIndices.forEach((index) => {
+    it(`should correctly find index "${index}" is migrated`, () => {
+      expect(checkIfMigratedIndexOutdated(index, indexVersionsByIndex, TEMPLATE_VERSION)).toBe(
+        false
+      );
+    });
+  });
+
+  it('should find non migrated index', () => {
+    expect(
+      checkIfMigratedIndexOutdated(
+        '.reindexed-v8-siem-signals-another-4-000001',
+        indexVersionsByIndex,
+        TEMPLATE_VERSION
+      )
+    ).toBe(true);
+  });
+});

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/migrations/get_non_migrated_signals_info.ts

@@ -17,6 +17,41 @@ import { isOutdated as getIsOutdated, signalsAreOutdated } from './helpers';
 import { getLatestIndexTemplateVersion } from './get_latest_index_template_version';
 import { getIndexAliasPerSpace } from './get_index_alias_per_space';
 
+const REINDEXED_PREFIX = '.reindexed-v8-';
+
+export const checkIfMigratedIndexOutdated = (
+  indexName: string,
+  indexVersionsByIndex: IndexVersionsByIndex,
+  latestTemplateVersion: number
+) => {
+  const isIndexOutdated = getIsOutdated({
+    current: indexVersionsByIndex[indexName] ?? 0,
+    target: latestTemplateVersion,
+  });
+
+  if (!isIndexOutdated) {
+    return false;
+  }
+
+  const nameWithoutPrefix = indexName.replace(REINDEXED_PREFIX, '.');
+
+  const hasOutdatedMigratedIndices = Object.entries(indexVersionsByIndex).every(
+    ([index, version]) => {
+      if (index === indexName) {
+        return true;
+      }
+
+      if (index.startsWith(nameWithoutPrefix) || index.startsWith(indexName)) {
+        return getIsOutdated({ current: version ?? 0, target: latestTemplateVersion });
+      }
+
+      return true;
+    }
+  );
+
+  return hasOutdatedMigratedIndices;
+};
+
 interface OutdatedSpaces {
   isMigrationRequired: boolean;
   spaces: string[];
@@ -85,6 +120,14 @@ export const getNonMigratedSignalsInfo = async ({
         const version = indexVersionsByIndex[indexName] ?? 0;
         const signalVersions = signalVersionsByIndex[indexName] ?? [];
 
+        // filter out migrated from 7.x to 8 indices
+        if (
+          indexName.startsWith(REINDEXED_PREFIX) &&
+          !checkIfMigratedIndexOutdated(indexName, indexVersionsByIndex, latestTemplateVersion)
+        ) {
+          return acc;
+        }
+
         const isOutdated =
           getIsOutdated({ current: version, target: latestTemplateVersion }) ||
           signalsAreOutdated({ signalVersions, target: latestTemplateVersion });

x-pack/solutions/security/plugins/security_solution/server/plugin.ts

@@ -45,6 +45,7 @@ import { AppClientFactory } from './client';
 import type { ConfigType } from './config';
 import { createConfig } from './config';
 import { initUiSettings } from './ui_settings';
+import { registerDeprecations } from './deprecations';
 import {
   APP_ID,
   APP_UI_ID,
@@ -212,6 +213,8 @@ export class Plugin implements ISecuritySolutionPlugin {
 
     this.ruleMonitoringService.setup(core, plugins);
 
+    registerDeprecations({ core, config: this.config, logger: this.logger });
+
     if (experimentalFeatures.riskScoringPersistence) {
       registerRiskScoringTask({
         getStartServices: core.getStartServices,

x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/data.json

@@ -0,0 +1,12 @@
+{
+    "type": "doc",
+    "value": {
+      "id": "1",
+      "index": ".siem-signals-another-space-legacy",
+      "source": {
+        "@timestamp": "2020-10-10T00:00:00.000Z",
+        "signal": {}
+      },
+      "type": "_doc"
+    }
+  }

x-pack/test/functional/es_archives/signals/legacy_signals_index_non_default_space/mappings.json

@@ -0,0 +1,29 @@
+{
+    "type": "index",
+    "value": {
+      "aliases": {
+        ".siem-signals-another-space": {
+          "is_write_index": false
+        }
+      },
+      "index": ".siem-signals-another-space-legacy",
+      "mappings": {
+        "_meta": {
+          "version": 1
+        },
+        "properties": {
+          "@timestamp": {
+            "type": "date"
+          },
+          "signal": { "type": "object" }
+        }
+      },
+      "settings": {
+        "index": {
+          "lifecycle": {
+            "indexing_complete": true
+          }
+        }
+      }
+    }
+  }