[Fleet] flag package policy SO to trigger agent policy bump (elastic#…

…200536) ## Summary Closes elastic#193352 Update: Using a new SO field `bump_agent_policy_revision` in package policy type to mark package policies for update, this will trigger an agent policy revision bump. The feature supports both legacy and new package policy SO types, and queries policies from all spaces. To test, add a model version change to the package policy type and save. After Fleet setup is run, the agent policies using the package policies should be bumped and deployed. The same effect can be achieved by manually updating a package policy SO and loading Fleet UI to trigger setup. ``` '2': { changes: [ { type: 'data_backfill', backfillFn: (doc) => { return { attributes: { ...doc.attributes, bump_agent_policy_revision: true } }; }, }, ], }, curl -sk -XPOST --user fleet_superuser:password -H 'content-type:application/json' \ -H'x-elastic-product-origin:fleet' \ http://localhost:9200/.kibana_ingest/_update_by_query -d ' { "query": { "match": { "type": "fleet-package-policies" } },"script": { "source": "ctx._source[\"fleet-package-policies\"].bump_agent_policy_revision = true", "lang": "painless" } }' ``` ``` [2024-11-20T14:40:30.064+01:00][INFO ][plugins.fleet] Found 1 package policies that need agent policy revision bump [2024-11-20T14:40:31.933+01:00][DEBUG][plugins.fleet] Updated 1 package policies in space space1 in 1869ms, bump 1 agent policies [2024-11-20T14:40:35.056+01:00][DEBUG][plugins.fleet] Deploying 1 policies [2024-11-20T14:40:35.493+01:00][DEBUG][plugins.fleet] Deploying policies: 7f108cf2-4cf0-4a11-8df4-fc69d00a3484:10 ``` TODO: - the same flag has to be added on agent policy and output types, and the task extended to update them - I plan to do this in another pr, so that this doesn't become too big - add integration test if possible ### Scale testing Tested with 500 agent policies split to 2 spaces, 1 integration per policy and bumping the flag in a new saved object model version, the bump task took about 6s. The deploy policies step is async, took about 30s. ``` [2024-11-20T15:53:55.628+01:00][INFO ][plugins.fleet] Found 501 package policies that need agent policy revision bump [2024-11-20T15:53:57.881+01:00][DEBUG][plugins.fleet] Updated 250 package policies in space space1 in 2253ms, bump 250 agent policies [2024-11-20T15:53:59.926+01:00][DEBUG][plugins.fleet] Updated 251 package policies in space default in 4298ms, bump 251 agent policies [2024-11-20T15:54:01.186+01:00][DEBUG][plugins.fleet] Deploying 250 policies [2024-11-20T15:54:29.989+01:00][DEBUG][plugins.fleet] Deploying policies: test-policy-space1-1:4, ... [2024-11-20T15:54:33.538+01:00][DEBUG][plugins.fleet] Deploying policies: policy-elastic-agent-on-cloud:4, test-policy-default-1:4, ... ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <[email protected]>
elena-shostak · Nov 25, 2024 · 973c695 · 973c695
1 parent a41017e
commit 973c695
Show file tree

Hide file tree

Showing 23 changed files with 392 additions and 89 deletions.
diff --git a/oas_docs/bundle.json b/oas_docs/bundle.json
@@ -36623,6 +36623,7 @@
                           "type": "boolean"
                         },
                         "use_space_awareness_migration_started_at": {
+                          "nullable": true,
                           "type": "string"
                         },
                         "use_space_awareness_migration_status": {
@@ -36824,6 +36825,7 @@
                           "type": "boolean"
                         },
                         "use_space_awareness_migration_started_at": {
+                          "nullable": true,
                           "type": "string"
                         },
                         "use_space_awareness_migration_status": {

diff --git a/oas_docs/bundle.serverless.json b/oas_docs/bundle.serverless.json
@@ -36623,6 +36623,7 @@
                           "type": "boolean"
                         },
                         "use_space_awareness_migration_started_at": {
+                          "nullable": true,
                           "type": "string"
                         },
                         "use_space_awareness_migration_status": {
@@ -36824,6 +36825,7 @@
                           "type": "boolean"
                         },
                         "use_space_awareness_migration_started_at": {
+                          "nullable": true,
                           "type": "string"
                         },
                         "use_space_awareness_migration_status": {

diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml
@@ -29839,6 +29839,7 @@ paths:
                       secret_storage_requirements_met:
                         type: boolean
                       use_space_awareness_migration_started_at:
+                        nullable: true
                         type: string
                       use_space_awareness_migration_status:
                         enum:
@@ -29972,6 +29973,7 @@ paths:
                       secret_storage_requirements_met:
                         type: boolean
                       use_space_awareness_migration_started_at:
+                        nullable: true
                         type: string
                       use_space_awareness_migration_status:
                         enum:

diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
@@ -32607,6 +32607,7 @@ paths:
                       secret_storage_requirements_met:
                         type: boolean
                       use_space_awareness_migration_started_at:
+                        nullable: true
                         type: string
                       use_space_awareness_migration_status:
                         enum:
@@ -32739,6 +32740,7 @@ paths:
                       secret_storage_requirements_met:
                         type: boolean
                       use_space_awareness_migration_started_at:
+                        nullable: true
                         type: string
                       use_space_awareness_migration_status:
                         enum:

diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json
@@ -511,6 +511,7 @@
   ],
   "fleet-message-signing-keys": [],
   "fleet-package-policies": [
+    "bump_agent_policy_revision",
     "created_at",
     "created_by",
     "description",
@@ -692,6 +693,7 @@
     "version"
   ],
   "ingest-package-policies": [
+    "bump_agent_policy_revision",
     "created_at",
     "created_by",
     "description",

diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1715,6 +1715,9 @@
   },
   "fleet-package-policies": {
     "properties": {
+      "bump_agent_policy_revision": {
+        "type": "boolean"
+      },
       "created_at": {
         "type": "date"
       },
@@ -2300,6 +2303,9 @@
   },
   "ingest-package-policies": {
     "properties": {
+      "bump_agent_policy_revision": {
+        "type": "boolean"
+      },
       "created_at": {
         "type": "date"
       },

diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -108,7 +108,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "fleet-agent-policies": "f57d3b70e4175a19a18f18ee72a379ceec82e1fc",
         "fleet-fleet-server-host": "69be15f6b6f2a2875ad3c7050ddea7a87f505417",
         "fleet-message-signing-keys": "93421f43fed2526b59092a4e3c65d64bc2266c0f",
-        "fleet-package-policies": "2f4d524adb49a5281d3af0b66bb3003ba0ff2e44",
+        "fleet-package-policies": "8be2cabfed89e103e0d413f2900e9cf6cd31bc68",
         "fleet-preconfiguration-deletion-record": "c52ea1e13c919afe8a5e8e3adbb7080980ecc08e",
         "fleet-proxy": "6cb688f0d2dd856400c1dbc998b28704ff70363d",
         "fleet-setup-lock": "0dc784792c79b5af5a6e6b5dcac06b0dbaa90bde",
@@ -124,7 +124,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "ingest-agent-policies": "5e95e539826a40ad08fd0c1d161da0a4d86ffc6d",
         "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d",
         "ingest-outputs": "55988d5f778bbe0e76caa7e6468707a0a056bdd8",
-        "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22",
+        "ingest-package-policies": "dfa7b1045a2667a822181f40f012786724492439",
         "ingest_manager_settings": "111a616eb72627c002029c19feb9e6c439a10505",
         "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83",
         "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad",

diff --git a/x-pack/plugins/fleet/dev_docs/space_awareness.md b/x-pack/plugins/fleet/dev_docs/space_awareness.md
@@ -15,7 +15,7 @@ xpack.fleet.enableExperimental: ['useSpaceAwareness', 'subfeaturePrivileges']
 After the feature flag is enabled you will have to do another step to opt-in for the feature, that call will migrate the current space agnostic saved objects to new space aware saved objects.
 
 ```shell
-curl -u elastic:changeme -XPOST "http://localhost:5601/internal/fleet/enable_space_awareness" -H "kbn-xsrf: reporting" -H 'elastic-api-version: 1'
+curl -u elastic:changeme -XPOST "http://localhost:5601/internal/fleet/enable_space_awareness" -H "kbn-xsrf: reporting" -H 'elastic-api-version: 1' -H 'x-elastic-internal-origin: 1'
 ```
 
 ## Space aware entities in Fleet

diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts b/x-pack/plugins/fleet/scripts/create_agent_policies/create_agent_policies.ts
@@ -9,8 +9,7 @@ import { ToolingLog } from '@kbn/tooling-log';
 import yargs from 'yargs';
 import { chunk } from 'lodash';
 
-import { LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../common/constants';
-import { LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common';
+import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common/constants';
 
 import { packagePolicyFixture } from './fixtures';
 
@@ -30,20 +29,18 @@ const printUsage = () =>
 
 const INDEX_BULK_OP = '{ "index":{ "_id": "{{id}}" } }\n';
 
+const space = 'default';
 function getPolicyId(idx: number | string) {
-  return `test-policy-${idx}`;
+  return `test-policy-${space}-${idx}`;
 }
 
 async function createAgentPoliciesDocsBulk(range: number[]) {
   const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64');
   const body = range
     .flatMap((idx) => [
-      INDEX_BULK_OP.replace(
-        /{{id}}/,
-        `${LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE}:${getPolicyId(idx)}`
-      ),
+      INDEX_BULK_OP.replace(/{{id}}/, `${AGENT_POLICY_SAVED_OBJECT_TYPE}:${getPolicyId(idx)}`),
       JSON.stringify({
-        [LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE]: {
+        [AGENT_POLICY_SAVED_OBJECT_TYPE]: {
           namespace: 'default',
           monitoring_enabled: ['logs', 'metrics', 'traces'],
           name: `Test Policy ${idx}`,
@@ -60,11 +57,11 @@ async function createAgentPoliciesDocsBulk(range: number[]) {
           schema_version: '1.1.1',
           is_protected: false,
         },
-        type: LEGACY_AGENT_POLICY_SAVED_OBJECT_TYPE,
+        namespaces: [space],
+        type: AGENT_POLICY_SAVED_OBJECT_TYPE,
         references: [],
         managed: false,
         coreMigrationVersion: '8.8.0',
-        typeMigrationVersion: '10.3.0',
         created_at: new Date().toISOString(),
         updated_at: new Date().toISOString(),
       }) + '\n',
@@ -81,7 +78,7 @@ async function createAgentPoliciesDocsBulk(range: number[]) {
   const data = await res.json();
 
   if (!data.items) {
-    logger.error('Error creating agent policies docs: ' + JSON.stringify(data));
+    logger.error('Error creating agent policy docs: ' + JSON.stringify(data));
     process.exit(1);
   }
   return data;
@@ -91,14 +88,14 @@ async function createEnrollmentToken(range: number[]) {
   const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64');
   const body = range
     .flatMap((idx) => [
-      INDEX_BULK_OP.replace(/{{id}}/, `test-enrollment-token-${idx}`),
+      INDEX_BULK_OP.replace(/{{id}}/, `test-enrollment-token-${space}-${idx}`),
       JSON.stringify({
         active: true,
         api_key_id: 'faketest123',
         api_key: 'test==',
         name: `Test Policy ${idx}`,
         policy_id: `${getPolicyId(idx)}`,
-        namespaces: [],
+        namespaces: [space],
         created_at: new Date().toISOString(),
       }) + '\n',
     ])
@@ -115,7 +112,7 @@ async function createEnrollmentToken(range: number[]) {
   const data = await res.json();
 
   if (!data.items) {
-    logger.error('Error creating agent policies docs: ' + JSON.stringify(data));
+    logger.error('Error creating enrollment key docs: ' + JSON.stringify(data));
     process.exit(1);
   }
   return data;
@@ -125,14 +122,12 @@ async function createPackagePolicies(range: number[]) {
   const auth = 'Basic ' + Buffer.from(ES_SUPERUSER + ':' + ES_PASSWORD).toString('base64');
   const body = range
     .flatMap((idx) => [
-      INDEX_BULK_OP.replace(
-        /{{id}}/,
-        `${LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE}:test-policy-${idx}`
-      ),
+      INDEX_BULK_OP.replace(/{{id}}/, `fleet-package-policies:test-policy-${space}-${idx}`),
       JSON.stringify(
         packagePolicyFixture({
           idx,
           agentPolicyId: getPolicyId(idx),
+          space,
         })
       ) + '\n',
     ])
@@ -150,7 +145,7 @@ async function createPackagePolicies(range: number[]) {
   const data = await res.json();
 
   if (!data.items) {
-    logger.error('Error creating agent policies docs: ' + JSON.stringify(data));
+    logger.error('Error creating package policy docs: ' + JSON.stringify(data));
     process.exit(1);
   }
   return data;

diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts b/x-pack/plugins/fleet/scripts/create_agent_policies/fixtures.ts
@@ -8,11 +8,13 @@
 export const packagePolicyFixture = ({
   agentPolicyId,
   idx,
+  space,
 }: {
   idx: number;
   agentPolicyId: string;
+  space: string;
 }) => ({
-  'ingest-package-policies': {
+  'fleet-package-policies': {
     name: `system-test-${idx}`,
     namespace: '',
     description: '',
@@ -790,11 +792,12 @@ export const packagePolicyFixture = ({
     updated_at: '2024-08-30T13:45:51.197Z',
     updated_by: 'system',
   },
-  type: 'ingest-package-policies',
+  namespaces: [space],
+  type: 'fleet-package-policies',
   references: [],
   managed: false,
   coreMigrationVersion: '8.8.0',
-  typeMigrationVersion: '10.14.0',
+  typeMigrationVersion: '10.1.0',
   updated_at: '2024-08-30T13:45:51.197Z',
   created_at: '2024-08-30T13:45:51.197Z',
 });
diff --git a/x-pack/plugins/fleet/scripts/create_agent_policies/index.js b/x-pack/plugins/fleet/scripts/create_agent_policies/index.js
@@ -12,6 +12,6 @@ require('./create_agent_policies').run();
 Usage:
 
 cd x-pack/plugins/fleet
-node scripts/create_agents/index.js
+node scripts/create_agent_policies/index.js
 
 */
diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts
@@ -143,6 +143,7 @@ import { registerFieldsMetadataExtractors } from './services/register_fields_met
 import { registerUpgradeManagedPackagePoliciesTask } from './services/setup/managed_package_policies';
 import { registerDeployAgentPoliciesTask } from './services/agent_policies/deploy_agent_policies_task';
 import { DeleteUnenrolledAgentsTask } from './tasks/delete_unenrolled_agents_task';
+import { registerBumpAgentPoliciesTask } from './services/agent_policies/bump_agent_policies_task';
 
 export interface FleetSetupDeps {
   security: SecurityPluginSetup;
@@ -619,6 +620,7 @@ export class FleetPlugin
     // Register task
     registerUpgradeManagedPackagePoliciesTask(deps.taskManager);
     registerDeployAgentPoliciesTask(deps.taskManager);
+    registerBumpAgentPoliciesTask(deps.taskManager);
 
     this.bulkActionsResolver = new BulkActionsResolver(deps.taskManager, core);
     this.checkDeletedFilesTask = new CheckDeletedFilesTask({

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -619,6 +619,7 @@ export const getSavedObjectTypes = (
           updated_by: { type: 'keyword' },
           created_at: { type: 'date' },
           created_by: { type: 'keyword' },
+          bump_agent_policy_revision: { type: 'boolean' },
         },
       },
       modelVersions: {
@@ -763,6 +764,16 @@ export const getSavedObjectTypes = (
             },
           ],
         },
+        '15': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                bump_agent_policy_revision: { type: 'boolean' },
+              },
+            },
+          ],
+        },
       },
       migrations: {
         '7.10.0': migratePackagePolicyToV7100,
@@ -823,6 +834,19 @@ export const getSavedObjectTypes = (
           updated_by: { type: 'keyword' },
           created_at: { type: 'date' },
           created_by: { type: 'keyword' },
+          bump_agent_policy_revision: { type: 'boolean' },
+        },
+      },
+      modelVersions: {
+        '1': {
+          changes: [
+            {
+              type: 'mappings_addition',
+              addedMappings: {
+                bump_agent_policy_revision: { type: 'boolean' },
+              },
+            },
+          ],
         },
       },
     },