[Defend Workflows] Enable FTR API tests for serverless using tags (el…

…astic#170418) ## Summary With a similar solution as elastic#169946, almost all of our FTR API tests (`x-pack/test/security_solution_endpoint_api_int/`) can now run against serverless, using `suiteTags` under the hood. ## Usage ```ts describe('This is the suite.', function() { // a custom function wraps the `this.tags()` function to provide type safety targetTags(this, ['@ess', '@serverless', '@skipInServerless']); }) ``` > **Note** > Only `describe()` blocks can be tagged. ## Changes: - serverless config is added for endpoint FTR API tests: `x-pack/test/security_solution_endpoint_api_int/serverless.config.ts` - roles are created only when running against ESS. when running against serverless, a subset of users/roles already exist (see [this list](https://github.com/elastic/kibana/pull/170418/files#diff-5aaeaeaedad4321151d5388437084b27f271a6254e95f0352c9a10c3126eddc8R54)) - tests that use roles that doesn't exist on serverless (`artifact_read_privileges`, `hunter`) are therefore skipped against serverless
crespocarlos · Nov 9, 2023 · 60c6490 · 60c6490
1 parent e42ee8a
commit 60c6490
Show file tree

Hide file tree

Showing 26 changed files with 374 additions and 221 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -16,6 +16,7 @@ disabled:
   - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
   - x-pack/test/security_solution_endpoint/config.base.ts
+  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -385,6 +386,7 @@ enabled:
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
   - x-pack/test/security_solution_endpoint_api_int/config.ts
+  - x-pack/test/security_solution_endpoint_api_int/serverless.config.ts
   - x-pack/test/security_solution_endpoint/endpoint.config.ts
   - x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/integrations.config.ts

diff --git a/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts b/x-pack/plugins/security_solution/scripts/endpoint/common/roles_users/index.ts
@@ -22,6 +22,7 @@ import {
 import { getDetectionsEngineer } from './detections_engineer';
 import { getWithResponseActionsRole } from './with_response_actions_role';
 import { getNoResponseActionsRole } from './without_response_actions_role';
+import { getWithArtifactReadPrivilegesRole } from './with_artifact_read_privileges_role';
 
 export * from './with_response_actions_role';
 export * from './without_response_actions_role';
@@ -74,6 +75,7 @@ export const ENDPOINT_SECURITY_ROLE_NAMES = Object.freeze({
   endpoint_response_actions_access: 'endpoint_response_actions_access',
   endpoint_response_actions_no_access: 'endpoint_response_actions_no_access',
   endpoint_security_policy_management_read: 'endpoint_security_policy_management_read',
+  artifact_read_privileges: 'artifact_read_privileges',
 });
 
 export const getAllEndpointSecurityRoles = (): EndpointSecurityRoleDefinitions => {
@@ -135,5 +137,9 @@ export const getAllEndpointSecurityRoles = (): EndpointSecurityRoleDefinitions =
       ...getEndpointSecurityPolicyManagementReadRole(),
       name: 'endpoint_security_policy_management_read',
     },
+    artifact_read_privileges: {
+      ...getWithArtifactReadPrivilegesRole(),
+      name: 'artifact_read_privileges',
+    },
   };
 };
diff --git a/x-pack/test/security_solution_endpoint/config.base.ts b/x-pack/test/security_solution_endpoint/config.base.ts
@@ -14,7 +14,10 @@ import {
 } from '../security_solution_endpoint_api_int/registry';
 import type { TargetTags } from './target_tags';
 
-const SUITE_TAGS: Record<string, { include: TargetTags[]; exclude: TargetTags[] }> = {
+export const SUITE_TAGS: Record<
+  'ess' | 'serverless',
+  { include: TargetTags[]; exclude: TargetTags[] }
+> = {
   ess: {
     include: ['@ess'],
     exclude: ['@skipInEss'],

diff --git a/x-pack/test/security_solution_endpoint/services/index.ts b/x-pack/test/security_solution_endpoint/services/index.ts
@@ -13,7 +13,10 @@ import { TimelineTestService } from '../../security_solution_ftr/services/timeli
 import { DetectionsTestService } from '../../security_solution_ftr/services/detections';
 import { EndpointPolicyTestResourcesProvider } from './endpoint_policy';
 import { EndpointArtifactsTestResources } from './endpoint_artifacts';
-import { KibanaSupertestWithCertProvider } from './supertest_with_cert';
+import {
+  KibanaSupertestWithCertProvider,
+  KibanaSupertestWithCertWithoutAuthProvider,
+} from './supertest_with_cert';
 
 export const services = {
   ...xPackFunctionalServices,
@@ -31,4 +34,5 @@ export const svlServices = {
   ...services,
 
   supertest: KibanaSupertestWithCertProvider,
+  supertestWithoutAuth: KibanaSupertestWithCertWithoutAuthProvider,
 };
diff --git a/x-pack/test/security_solution_endpoint/services/supertest_with_cert.ts b/x-pack/test/security_solution_endpoint/services/supertest_with_cert.ts
@@ -15,3 +15,11 @@ export function KibanaSupertestWithCertProvider({ getService }: FtrProviderConte
 
   return supertest.agent(kibanaServerUrl, { ca });
 }
+
+export function KibanaSupertestWithCertWithoutAuthProvider({ getService }: FtrProviderContext) {
+  const config = getService('config');
+  const kibanaServerUrl = formatUrl({ ...config.get('servers.kibana'), auth: false });
+  const ca = config.get('servers.kibana').certificateAuthorities;
+
+  return supertest.agent(kibanaServerUrl, { ca });
+}
diff --git a/x-pack/test/security_solution_endpoint/target_tags.ts b/x-pack/test/security_solution_endpoint/target_tags.ts
@@ -5,13 +5,44 @@
  * 2.0.
  */
 
-export type TargetTags =
-  | '@ess'
-  | '@skipInEss'
-  | '@serverless'
-  | '@skipInServerless'
-  | '@brokenInServerless';
-
-export const targetTags = (thisSuite: Mocha.Suite, tags: TargetTags[]) => {
+import expect from '@kbn/expect';
+
+const TARGET_TAGS = [
+  '@ess',
+  '@skipInEss',
+  '@serverless',
+  '@skipInServerless',
+  '@brokenInServerless',
+] as const;
+
+export type TargetTags = typeof TARGET_TAGS[number];
+
+export function targetTags(thisSuite: Mocha.Suite, tags: TargetTags[]) {
+  // @ts-ignore: _tags is not publicly visible
+  const existingTags = (thisSuite._tags as string[]) ?? [];
+  const existingTargetTags = existingTags.filter((tag) => TARGET_TAGS.includes(tag as TargetTags));
+
+  if (existingTargetTags.length > 0) {
+    return expect().fail(`
+    
+    ⚠️  ERROR in \`${targetTags.name}()\`: the passed suite already has target tags.
+
+       Suite name:         ${thisSuite.title}
+       Existing tags:      ${existingTargetTags.join(', ')}
+       New tags:           ${tags.join(', ')}
+
+    💡 This can happen if you call \`${targetTags.name}()\` twice in the same block, or
+        → from the inside of an arrow function
+        → which is passed to a \`describe()\` block
+        → which is somewhere inside \`${thisSuite.title}\`.
+
+    ☝️  Correct usage:
+          describe('must receive a regular function', function () {
+            ${targetTags.name}(this, ['@serverless']);
+          })
+    
+    `);
+  }
+
   thisSuite.tags(tags);
-};
+}
diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/blocklists.ts b/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/blocklists.ts
@@ -13,6 +13,7 @@ import {
   GLOBAL_ARTIFACT_TAG,
 } from '@kbn/security-solution-plugin/common/endpoint/service/artifacts';
 import { ExceptionsListItemGenerator } from '@kbn/security-solution-plugin/common/endpoint/data_generators/exceptions_list_item_generator';
+import { targetTags } from '../../../security_solution_endpoint/target_tags';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { PolicyTestResourceInfo } from '../../../security_solution_endpoint/services/endpoint_policy';
 import { ArtifactTestData } from '../../../security_solution_endpoint/services/endpoint_artifacts';
@@ -24,7 +25,9 @@ export default function ({ getService }: FtrProviderContext) {
   const endpointPolicyTestResources = getService('endpointPolicyTestResources');
   const endpointArtifactTestResources = getService('endpointArtifactTestResources');
 
-  describe('Endpoint artifacts (via lists plugin): Blocklists', () => {
+  describe('Endpoint artifacts (via lists plugin): Blocklists', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     let fleetEndpointPolicy: PolicyTestResourceInfo;
 
     before(async () => {
@@ -155,7 +158,7 @@ export default function ({ getService }: FtrProviderContext) {
 
             body.entries[0].field = 'some.invalid.field';
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.analyst_hunter, 'changeme')
+              .auth(ROLE.endpoint_policy_manager, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(body)
               .expect(400)
@@ -176,7 +179,7 @@ export default function ({ getService }: FtrProviderContext) {
             ];
 
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.analyst_hunter, 'changeme')
+              .auth(ROLE.endpoint_policy_manager, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(body)
               .expect(400)
@@ -197,7 +200,7 @@ export default function ({ getService }: FtrProviderContext) {
             ];
 
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.analyst_hunter, 'changeme')
+              .auth(ROLE.endpoint_policy_manager, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(body)
               .expect(400)
@@ -224,7 +227,7 @@ export default function ({ getService }: FtrProviderContext) {
             ];
 
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.analyst_hunter, 'changeme')
+              .auth(ROLE.endpoint_policy_manager, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(body)
               .expect(400)
@@ -258,7 +261,7 @@ export default function ({ getService }: FtrProviderContext) {
             ];
 
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.analyst_hunter, 'changeme')
+              .auth(ROLE.endpoint_policy_manager, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(body)
               .expect(400)
@@ -272,7 +275,7 @@ export default function ({ getService }: FtrProviderContext) {
             body.os_types = ['linux', 'windows'];
 
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.analyst_hunter, 'changeme')
+              .auth(ROLE.endpoint_policy_manager, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(body)
               .expect(400)
@@ -297,32 +300,31 @@ export default function ({ getService }: FtrProviderContext) {
         for (const blocklistApiCall of [...needsWritePrivilege, ...needsReadPrivilege]) {
           it(`should not error on [${blocklistApiCall.method}] - [${blocklistApiCall.info}]`, async () => {
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.analyst_hunter, 'changeme')
+              .auth(ROLE.endpoint_policy_manager, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(blocklistApiCall.getBody())
               .expect(200);
           });
         }
       });
 
-      describe('and user has authorization to read blocklist', () => {
+      describe('and user has authorization to read blocklist', function () {
+        targetTags(this, ['@skipInServerless']); // no such role in serverless
+
         for (const blocklistApiCall of [...blocklistApiCalls, ...needsWritePrivilege]) {
           it(`should error on [${blocklistApiCall.method}] - [${blocklistApiCall.info}]`, async () => {
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.artifact_read_role, 'changeme')
+              .auth(ROLE.artifact_read_privileges, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(blocklistApiCall.getBody())
-              .expect(403, {
-                status_code: 403,
-                message: 'EndpointArtifactError: Endpoint authorization failure',
-              });
+              .expect(403);
           });
         }
 
         for (const blocklistApiCall of needsReadPrivilege) {
           it(`should not error on [${blocklistApiCall.method}] - [${blocklistApiCall.info}]`, async () => {
             await supertestWithoutAuth[blocklistApiCall.method](blocklistApiCall.path)
-              .auth(ROLE.artifact_read_role, 'changeme')
+              .auth(ROLE.artifact_read_privileges, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(blocklistApiCall.getBody())
               .expect(200);
@@ -341,10 +343,7 @@ export default function ({ getService }: FtrProviderContext) {
               .auth(ROLE.t1_analyst, 'changeme')
               .set('kbn-xsrf', 'true')
               .send(blocklistApiCall.getBody())
-              .expect(403, {
-                status_code: 403,
-                message: 'EndpointArtifactError: Endpoint authorization failure',
-              });
+              .expect(403);
           });
         }
       });

diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/event_filters.ts b/x-pack/test/security_solution_endpoint_api_int/apis/endpoint_artifacts/event_filters.ts
@@ -14,6 +14,7 @@ import {
   getImportExceptionsListSchemaMock,
   toNdJsonString,
 } from '@kbn/lists-plugin/common/schemas/request/import_exceptions_schema.mock';
+import { targetTags } from '../../../security_solution_endpoint/target_tags';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { PolicyTestResourceInfo } from '../../../security_solution_endpoint/services/endpoint_policy';
 import { ArtifactTestData } from '../../../security_solution_endpoint/services/endpoint_artifacts';
@@ -25,7 +26,9 @@ export default function ({ getService }: FtrProviderContext) {
   const endpointPolicyTestResources = getService('endpointPolicyTestResources');
   const endpointArtifactTestResources = getService('endpointArtifactTestResources');
 
-  describe('Endpoint artifacts (via lists plugin): Event Filters', () => {
+  describe('Endpoint artifacts (via lists plugin): Event Filters', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     let fleetEndpointPolicy: PolicyTestResourceInfo;
 
     before(async () => {
@@ -182,7 +185,7 @@ export default function ({ getService }: FtrProviderContext) {
           const body = eventFilterApiCall.getBody({ os_types: ['linux', 'windows'] });
 
           await supertestWithoutAuth[eventFilterApiCall.method](eventFilterApiCall.path)
-            .auth(ROLE.endpoint_security_policy_manager, 'changeme')
+            .auth(ROLE.endpoint_policy_manager, 'changeme')
             .set('kbn-xsrf', 'true')
             .send(body)
             .expect(400)
@@ -197,7 +200,7 @@ export default function ({ getService }: FtrProviderContext) {
 
           // Using superuser there as we need custom license for this action
           await supertest[eventFilterApiCall.method](eventFilterApiCall.path)
-            .auth(ROLE.endpoint_security_policy_manager, 'changeme')
+            .auth(ROLE.endpoint_policy_manager, 'changeme')
             .set('kbn-xsrf', 'true')
             .send(body)
             .expect(400)
@@ -210,7 +213,7 @@ export default function ({ getService }: FtrProviderContext) {
 
           // Using superuser here as we need custom license for this action
           await supertest[eventFilterApiCall.method](eventFilterApiCall.path)
-            .auth(ROLE.endpoint_security_policy_manager, 'changeme')
+            .auth(ROLE.endpoint_policy_manager, 'changeme')
             .set('kbn-xsrf', 'true')
             .send(body)
             .expect(200);
@@ -222,32 +225,31 @@ export default function ({ getService }: FtrProviderContext) {
       for (const eventFilterApiCall of [...needsWritePrivilege, ...needsReadPrivilege]) {
         it(`should not error on [${eventFilterApiCall.method}] - [${eventFilterApiCall.info}]`, async () => {
           await supertestWithoutAuth[eventFilterApiCall.method](eventFilterApiCall.path)
-            .auth(ROLE.endpoint_security_policy_manager, 'changeme')
+            .auth(ROLE.endpoint_policy_manager, 'changeme')
             .set('kbn-xsrf', 'true')
             .send(eventFilterApiCall.getBody())
             .expect(200);
         });
       }
     });
 
-    describe('and user has authorization to read event filters', () => {
+    describe('and user has authorization to read event filters', function () {
+      targetTags(this, ['@skipInServerless']); // no such role in serverless
+
       for (const eventFilterApiCall of [...eventFilterCalls, ...needsWritePrivilege]) {
         it(`should error on [${eventFilterApiCall.method}] - [${eventFilterApiCall.info}]`, async () => {
           await supertestWithoutAuth[eventFilterApiCall.method](eventFilterApiCall.path)
-            .auth(ROLE.artifact_read_role, 'changeme')
+            .auth(ROLE.hunter, 'changeme')
             .set('kbn-xsrf', 'true')
             .send(eventFilterApiCall.getBody())
-            .expect(403, {
-              status_code: 403,
-              message: 'EndpointArtifactError: Endpoint authorization failure',
-            });
+            .expect(403);
         });
       }
 
       for (const eventFilterApiCall of needsReadPrivilege) {
         it(`should not error on [${eventFilterApiCall.method}] - [${eventFilterApiCall.info}]`, async () => {
           await supertestWithoutAuth[eventFilterApiCall.method](eventFilterApiCall.path)
-            .auth(ROLE.artifact_read_role, 'changeme')
+            .auth(ROLE.hunter, 'changeme')
             .set('kbn-xsrf', 'true')
             .send(eventFilterApiCall.getBody())
             .expect(200);
@@ -266,10 +268,7 @@ export default function ({ getService }: FtrProviderContext) {
             .auth(ROLE.t1_analyst, 'changeme')
             .set('kbn-xsrf', 'true')
             .send(eventFilterApiCall.getBody())
-            .expect(403, {
-              status_code: 403,
-              message: 'EndpointArtifactError: Endpoint authorization failure',
-            });
+            .expect(403);
         });
       }
     });