[Fleet] Telemetry for space awareness (#206493)

x-pack/platform/plugins/shared/fleet/server/collectors/agent_policies.ts

@@ -11,11 +11,13 @@ import _ from 'lodash';
 import { OUTPUT_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../common';
 import type { OutputSOAttributes, AgentPolicy } from '../types';
 import { getAgentPolicySavedObjectType } from '../services/agent_policy';
+import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server';
 
 export interface AgentPoliciesUsage {
   count: number;
   output_types: string[];
   count_with_global_data_tags: number;
+  count_with_non_default_space: number;
   avg_number_global_data_tags_per_policy?: number;
 }
 
@@ -33,17 +35,27 @@ export const getAgentPoliciesUsage = async (
   const outputsById = _.keyBy(outputs, 'id');
 
   const agentPolicySavedObjectType = await getAgentPolicySavedObjectType();
-  const { saved_objects: agentPolicies, total: totalAgentPolicies } =
-    await soClient.find<AgentPolicy>({
-      type: agentPolicySavedObjectType,
-      page: 1,
-      perPage: SO_SEARCH_LIMIT,
-    });
+  const { saved_objects: agentPolicies, total: totalAgentPolicies } = await soClient.find<
+    Pick<AgentPolicy, 'data_output_id' | 'monitoring_output_id' | 'global_data_tags'>
+  >({
+    type: agentPolicySavedObjectType,
+    page: 1,
+    perPage: SO_SEARCH_LIMIT,
+    namespaces: ['*'],
+    fields: ['monitoring_output_id', 'data_output_id', 'global_data_tags'],
+  });
 
+  let countWithNonDefaultSpace = 0;
   const uniqueOutputIds = new Set<string>();
   agentPolicies.forEach((agentPolicy) => {
-    uniqueOutputIds.add(agentPolicy.attributes.monitoring_output_id || defaultOutputId);
-    uniqueOutputIds.add(agentPolicy.attributes.data_output_id || defaultOutputId);
+    if (
+      (agentPolicy.namespaces?.length ?? 0) > 0 &&
+      agentPolicy.namespaces?.some((namespace) => namespace !== DEFAULT_NAMESPACE_STRING)
+    ) {
+      countWithNonDefaultSpace++;
+    }
+    uniqueOutputIds.add(agentPolicy.attributes?.monitoring_output_id || defaultOutputId);
+    uniqueOutputIds.add(agentPolicy.attributes?.data_output_id || defaultOutputId);
   });
 
   const uniqueOutputTypes = new Set(
@@ -56,10 +68,10 @@ export const getAgentPoliciesUsage = async (
 
   const [policiesWithGlobalDataTag, totalNumberOfGlobalDataTagFields] = agentPolicies.reduce(
     ([policiesNumber, fieldsNumber], agentPolicy) => {
-      if (agentPolicy.attributes.global_data_tags?.length ?? 0 > 0) {
+      if (agentPolicy.attributes?.global_data_tags?.length ?? 0 > 0) {
         return [
           policiesNumber + 1,
-          fieldsNumber + (agentPolicy.attributes.global_data_tags?.length ?? 0),
+          fieldsNumber + (agentPolicy.attributes?.global_data_tags?.length ?? 0),
         ];
       }
       return [policiesNumber, fieldsNumber];
@@ -70,6 +82,7 @@ export const getAgentPoliciesUsage = async (
   return {
     count: totalAgentPolicies,
     output_types: Array.from(uniqueOutputTypes),
+    count_with_non_default_space: countWithNonDefaultSpace,
     count_with_global_data_tags: policiesWithGlobalDataTag,
     avg_number_global_data_tags_per_policy:
       policiesWithGlobalDataTag > 0

x-pack/platform/plugins/shared/fleet/server/collectors/register.ts

@@ -38,7 +38,12 @@ export interface Usage {
 
 export interface FleetUsage extends Usage, AgentData {
   fleet_server_config: { policies: Array<{ input_config: any }> };
-  agent_policies: { count: number; output_types: string[] };
+  agent_policies: {
+    count: number;
+    output_types: string[];
+    count_with_global_data_tags: number;
+    count_with_non_default_space: number;
+  };
   agent_logs_panics_last_hour: AgentPanicLogsData['agent_logs_panics_last_hour'];
   agent_logs_top_errors?: string[];
   fleet_server_logs_top_errors?: string[];
@@ -55,6 +60,7 @@ export const fetchFleetUsage = async (
   if (!soClient || !esClient) {
     return;
   }
+
   const usage = {
     agents_enabled: getIsAgentsEnabled(config),
     agents: await getAgentUsage(soClient, esClient),

x-pack/platform/plugins/shared/fleet/server/integration_tests/fleet_usage_telemetry.test.ts

@@ -589,6 +589,7 @@ describe('fleet usage telemetry', () => {
           count: 3,
           output_types: expect.arrayContaining(['elasticsearch', 'logstash', 'third_type']),
           count_with_global_data_tags: 2,
+          count_with_non_default_space: 0,
           avg_number_global_data_tags_per_policy: 2,
         },
         agent_logs_panics_last_hour: [

x-pack/platform/plugins/shared/fleet/server/plugin.ts

@@ -119,6 +119,7 @@ import {
 import {
   fetchAgentsUsage,
   fetchFleetUsage,
+  type FleetUsage,
   registerFleetUsageCollector,
 } from './collectors/register';
 import { FleetArtifactsClient } from './services/artifacts';
@@ -198,6 +199,7 @@ export interface FleetAppContext {
   unenrollInactiveAgentsTask: UnenrollInactiveAgentsTask;
   deleteUnenrolledAgentsTask: DeleteUnenrolledAgentsTask;
   taskManagerStart?: TaskManagerStartContract;
+  fetchUsage?: (abortController: AbortController) => Promise<FleetUsage | undefined>;
 }
 
 export type FleetSetupContract = void;
@@ -301,6 +303,7 @@ export class FleetPlugin
   private packageService?: PackageService;
   private packagePolicyService?: PackagePolicyService;
   private policyWatcher?: PolicyWatcher;
+  private fetchUsage?: (abortController: AbortController) => Promise<FleetUsage | undefined>;
 
   constructor(private readonly initializerContext: PluginInitializerContext) {
     this.config$ = this.initializerContext.config.create<FleetConfigType>();
@@ -603,9 +606,9 @@ export class FleetPlugin
 
     // Register usage collection
     registerFleetUsageCollector(core, config, deps.usageCollection);
-    const fetch = async (abortController: AbortController) =>
+    this.fetchUsage = async (abortController: AbortController) =>
       await fetchFleetUsage(core, config, abortController);
-    this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, fetch);
+    this.fleetUsageSender = new FleetUsageSender(deps.taskManager, core, this.fetchUsage);
     registerFleetUsageLogger(deps.taskManager, async () => fetchAgentsUsage(core, config));
 
     const fetchAgents = async (abortController: AbortController) =>
@@ -694,6 +697,7 @@ export class FleetPlugin
       unenrollInactiveAgentsTask: this.unenrollInactiveAgentsTask!,
       deleteUnenrolledAgentsTask: this.deleteUnenrolledAgentsTask!,
       taskManagerStart: plugins.taskManager,
+      fetchUsage: this.fetchUsage,
     });
     licenseService.start(plugins.licensing.license$);
     this.telemetryEventsSender.start(plugins.telemetry, core).catch(() => {});

x-pack/platform/plugins/shared/fleet/server/routes/app/index.ts

@@ -189,7 +189,27 @@ export const GenerateServiceTokenResponseSchema = schema.object({
 
 export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType) => {
   const experimentalFeatures = parseExperimentalConfigValue(config.enableExperimental);
-
+  router.versioned
+    .get({
+      path: '/internal/fleet/telemetry/usage',
+      access: 'internal',
+      security: {
+        authz: {
+          requiredPrivileges: [
+            FLEET_API_PRIVILEGES.AGENTS.ALL,
+            FLEET_API_PRIVILEGES.AGENT_POLICIES.ALL,
+            FLEET_API_PRIVILEGES.SETTINGS.ALL,
+          ],
+        },
+      },
+    })
+    .addVersion(
+      {
+        version: API_VERSIONS.internal.v1,
+        validate: {},
+      },
+      getTelemetryUsageHandler
+    );
   if (experimentalFeatures.useSpaceAwareness) {
     router.versioned
       .post({
@@ -288,3 +308,16 @@ export const registerRoutes = (router: FleetAuthzRouter, config: FleetConfigType
       generateServiceTokenHandler
     );
 };
+const getTelemetryUsageHandler: FleetRequestHandler = async (context, request, response) => {
+  const fetchUsage = appContextService.getFetchUsage();
+  if (!fetchUsage) {
+    throw new Error('Fetch usage is not initialized.');
+  }
+  const usage = await fetchUsage(new AbortController());
+
+  return response.ok({
+    body: {
+      usage,
+    },
+  });
+};

x-pack/platform/plugins/shared/fleet/server/services/app_context.ts

@@ -52,6 +52,7 @@ import type { MessageSigningServiceInterface } from '..';
 
 import type { BulkActionsResolver } from './agents/bulk_actions_resolver';
 import { type UninstallTokenServiceInterface } from './security/uninstall_token_service';
+import type { FleetUsage } from '../collectors/register';
 
 class AppContextService {
   private encryptedSavedObjects: EncryptedSavedObjectsClient | undefined;
@@ -80,6 +81,7 @@ class AppContextService {
   private messageSigningService: MessageSigningServiceInterface | undefined;
   private uninstallTokenService: UninstallTokenServiceInterface | undefined;
   private taskManagerStart: TaskManagerStartContract | undefined;
+  private fetchUsage?: (abortController: AbortController) => Promise<FleetUsage | undefined>;
 
   public start(appContext: FleetAppContext) {
     this.data = appContext.data;
@@ -105,6 +107,7 @@ class AppContextService {
     this.messageSigningService = appContext.messageSigningService;
     this.uninstallTokenService = appContext.uninstallTokenService;
     this.taskManagerStart = appContext.taskManagerStart;
+    this.fetchUsage = appContext.fetchUsage;
 
     if (appContext.config$) {
       this.config$ = appContext.config$;
@@ -344,6 +347,10 @@ class AppContextService {
   public getUninstallTokenService() {
     return this.uninstallTokenService;
   }
+
+  public getFetchUsage() {
+    return this.fetchUsage;
+  }
 }
 
 export const appContextService = new AppContextService();

x-pack/platform/plugins/shared/fleet/server/services/telemetry/fleet_usages_schema.ts

@@ -352,6 +352,12 @@ export const fleetUsagesSchema: RootSchema<any> = {
           description: 'Number of agent policies using global data tags',
         },
       },
+      count_with_non_default_space: {
+        type: 'long',
+        _meta: {
+          description: 'Number of agent policies using another space than the default one',
+        },
+      },
       avg_number_global_data_tags_per_policy: {
         type: 'long',
         _meta: {

x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts

@@ -45,6 +45,7 @@ import {
   GetUninstallTokensMetadataResponse,
 } from '@kbn/fleet-plugin/common/types/rest_spec/uninstall_token';
 import { SimplifiedPackagePolicy } from '@kbn/fleet-plugin/common/services/simplified_package_policy_helper';
+import { type FleetUsage } from '@kbn/fleet-plugin/server/collectors/register';
 import { testUsers } from '../test_users';
 
 export class SpaceTestApiClient {
@@ -375,6 +376,16 @@ export class SpaceTestApiClient {
 
     return res;
   }
+  // Fleet Usage
+  async getFleetUsage(spaceId?: string): Promise<{ usage: FleetUsage }> {
+    const { body: res } = await this.supertest
+      .get(`${this.getBaseUrl(spaceId)}/internal/fleet/telemetry/usage`)
+      .set('kbn-xsrf', 'xxxx')
+      .set('elastic-api-version', '1')
+      .expect(200);
+
+    return res;
+  }
   // Space Settings
   async getSpaceSettings(spaceId?: string): Promise<GetSpaceSettingsResponse> {
     const { body: res } = await this.supertest

x-pack/test/fleet_api_integration/apis/space_awareness/index.js

@@ -18,5 +18,6 @@ export default function loadTests({ loadTestFile }) {
     loadTestFile(require.resolve('./actions'));
     loadTestFile(require.resolve('./change_space_agent_policies'));
     loadTestFile(require.resolve('./space_awareness_migration'));
+    loadTestFile(require.resolve('./telemetry'));
   });
 }

x-pack/test/fleet_api_integration/apis/space_awareness/telemetry.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { SpaceTestApiClient } from './api_helper';
+
+export default function (providerContext: FtrProviderContext) {
+  const { getService } = providerContext;
+  const supertest = getService('supertest');
+  const kibanaServer = getService('kibanaServer');
+  const spaces = getService('spaces');
+  let TEST_SPACE_1: string;
+
+  const apiClient = new SpaceTestApiClient(supertest);
+
+  describe('space_telemetry', function () {
+    before(async () => {
+      TEST_SPACE_1 = spaces.getDefaultTestSpace();
+      await kibanaServer.savedObjects.cleanStandardList();
+      await kibanaServer.savedObjects.cleanStandardList({
+        space: TEST_SPACE_1,
+      });
+      await spaces.createTestSpace(TEST_SPACE_1);
+      await apiClient.postEnableSpaceAwareness();
+      await Promise.all([
+        apiClient.createAgentPolicy(),
+        apiClient.createAgentPolicy(),
+        apiClient.createAgentPolicy(TEST_SPACE_1),
+        apiClient.createAgentPolicy(TEST_SPACE_1),
+        apiClient.createAgentPolicy(TEST_SPACE_1),
+      ]);
+    });
+
+    after(async () => {
+      await kibanaServer.savedObjects.cleanStandardList();
+      await kibanaServer.savedObjects.cleanStandardList({
+        space: TEST_SPACE_1,
+      });
+    });
+
+    it('return correct fleet usage', async () => {
+      const res = await apiClient.getFleetUsage();
+      expect(res.usage.agent_policies.count).to.eql(5);
+      expect(res.usage.agent_policies.count_with_non_default_space).to.eql(3);
+    });
+  });
+}