From bd9254fe1ebbc042968846dbd1fd52f1769dc0bb Mon Sep 17 00:00:00 2001 From: Abdul Wahab Zahid Date: Mon, 30 Dec 2024 12:46:49 +0100 Subject: [PATCH] [Dataset Quality] Fix failing es-promotion test on `8.x` (#205188) Newer builds of ES ingest `event.ingested` and `tags` fields into APM indices making the Dataset Quality tests fail which rely on ingested field counts against set number of field limits. The PR addresses this by explicitly ingesting the fields beforehand. After the merge to main, the PR will be backported to `8.x` to unblock es-promotion. --- .../src/lib/apm/apm_fields.ts | 2 +- .../src/lib/logs/index.ts | 1 + .../apps/dataset_quality/data/logs_data.ts | 4 ++++ .../dataset_quality/degraded_field_flyout.ts | 19 +++++++++++------ .../dataset_quality/degraded_field_flyout.ts | 21 +++++++++++-------- 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts b/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts index 1c9c379dbef56..3371cc27227cb 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/apm/apm_fields.ts @@ -126,7 +126,7 @@ export type ApmFields = Fields<{ 'error.id': string; 'error.type': string; 'error.culprit': string; - 'event.ingested': number; + 'event.ingested': string; 'event.name': string; 'event.action': string; 'event.outcome': string; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts index e4e382860767d..fa1856aa05ca3 100644 --- a/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts +++ b/packages/kbn-apm-synthtrace-client/src/lib/logs/index.ts @@ -36,6 +36,7 @@ export type LogDocument = Fields & 'error.message'?: string; 'event.original'?: string; 'event.dataset': string; + 'event.ingested': string; 'log.level'?: string; 'host.name'?: string; 'container.id'?: string; diff --git a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts index 3692a17709a2e..cd1668484337e 100644 --- a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts +++ b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts @@ -217,3 +217,7 @@ export const MORE_THAN_1024_CHARS = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; export const ANOTHER_1024_CHARS = 'grape fig tangerine tangerine kiwi lemon papaya cherry nectarine papaya mango cherry nectarine fig cherry fig grape mango mango quince fig strawberry mango quince date kiwi quince raspberry apple kiwi banana quince fig papaya grape mango cherry banana mango cherry lemon cherry tangerine fig quince quince papaya tangerine grape strawberry banana kiwi grape mango papaya nectarine banana nectarine kiwi papaya lemon apple lemon orange fig cherry grape apple nectarine papaya orange fig papaya date mango papaya mango cherry tangerine papaya apple banana papaya cherry strawberry grape raspberry lemon date papaya mango kiwi cherry fig banana banana apple date strawberry mango tangerine date lemon kiwi quince date orange orange papaya date apple fig tangerine quince tangerine date papaya banana banana orange raspberry papaya apple nectarine lemon raspberry raspberry mango cherry kiwi cherry cherry nectarine cherry date strawberry banana orange mango mango tangerine quince papaya papaya kiwi papaya strawberry date mango'; + +export const CONSISTENT_TAGS = [ + 'this_is_here_to_remove_variance_introduced_by_the_geoip_processor', +]; diff --git a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts index 19a2001603925..3a24fdd59eac3 100644 --- a/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts @@ -15,6 +15,7 @@ import { getInitialTestLogs, ANOTHER_1024_CHARS, MORE_THAN_1024_CHARS, + CONSISTENT_TAGS, } from './data'; import { logsSynthMappings } from './custom_mappings/custom_synth_mappings'; import { logsNginxMappings } from './custom_mappings/custom_integration_mappings'; @@ -187,7 +188,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .timestamp(timestamp) ); }), - // Ingest Degraded Logs with 26 fields in Apm DataSet + // Ingest Degraded Logs with 27 fields in Apm DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -205,7 +206,9 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .defaults({ 'service.name': serviceName, 'trace.id': generateShortId(), + 'event.ingested': new Date().toISOString(), test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + tags: CONSISTENT_TAGS, // To account for ES inserted error tags }) .timestamp(timestamp) ); @@ -227,7 +230,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid // Set Limit of 26 await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, { - 'mapping.total_fields.limit': 25, + 'mapping.total_fields.limit': 26, }); await synthtrace.index([ @@ -279,7 +282,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .timestamp(timestamp) ); }), - // Ingest Degraded Logs with 27 fields in Apm APP DataSet + // Ingest Degraded Logs with 29 fields in Apm APP DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -299,6 +302,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], 'cloud.project.id': generateShortId(), + 'event.ingested': new Date().toISOString(), + tags: CONSISTENT_TAGS, // To account for ES inserted error tags }) .timestamp(timestamp) ); @@ -330,13 +335,13 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid } ); - // Set Limit of 27 + // Set Limit of 28 await PageObjects.datasetQuality.setDataStreamSettings( PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ dataset: apmAppDatasetName, }) + '-000002', { - 'mapping.total_fields.limit': 27, + 'mapping.total_fields.limit': 28, } ); @@ -389,7 +394,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid .timestamp(timestamp) ); }), - // Ingest Degraded Logs with 27 fields in Apm APP DataSet + // Ingest Degraded Logs with 29 fields in Apm APP DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -409,6 +414,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], 'cloud.project.id': generateShortId(), + 'event.ingested': new Date().toISOString(), + tags: CONSISTENT_TAGS, // To account for ES inserted error tags }) .timestamp(timestamp) ); diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts index 4152a745d8541..4c4a8bb5eeb81 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/degraded_field_flyout.ts @@ -102,7 +102,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - describe('testing root cause for ignored fields', () => { + describe('detecting root cause for ignored fields', () => { before(async () => { // Create custom component template await synthtrace.createComponentTemplate( @@ -134,14 +134,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Install Nginx Integration and ingest logs for it await PageObjects.observabilityLogsExplorer.installPackage(nginxPkg); - // Create custom component template to avoid issues with LogsDB + // Create custom component template for Nginx to avoid issues with LogsDB await synthtrace.createComponentTemplate( customComponentTemplateNameNginx, logsNginxMappings(nginxAccessDatasetName) ); await synthtrace.index([ - // Ingest Degraded Logs with 25 fields + // Ingest Degraded Logs with 25 fields in degraded DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -187,7 +187,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .timestamp(timestamp) ); }), - // Ingest Degraded Logs with 26 fields in Apm DataSet + // Ingest Degraded Logs with 27 fields in Apm DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -206,6 +206,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'service.name': serviceName, 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], + 'event.ingested': new Date().toISOString(), // this works around a geoip limitation in CI tags: CONSISTENT_TAGS, }) @@ -229,7 +230,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // Set Limit of 26 await PageObjects.datasetQuality.setDataStreamSettings(apmAppDataStreamName, { - 'mapping.total_fields.limit': 25, + 'mapping.total_fields.limit': 26, }); await synthtrace.index([ @@ -281,7 +282,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .timestamp(timestamp) ); }), - // Ingest Degraded Logs with 27 fields in Apm APP DataSet + // Ingest Degraded Logs with 29 fields in Apm APP DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -303,6 +304,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'cloud.project.id': generateShortId(), // this works around a geoip limitation in CI tags: CONSISTENT_TAGS, + 'event.ingested': new Date().toISOString(), }) .timestamp(timestamp) ); @@ -334,13 +336,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { } ); - // Set Limit of 27 + // Set Limit of 28 await PageObjects.datasetQuality.setDataStreamSettings( PageObjects.datasetQuality.generateBackingIndexNameWithoutVersion({ dataset: apmAppDatasetName, }) + '-000002', { - 'mapping.total_fields.limit': 27, + 'mapping.total_fields.limit': 28, } ); @@ -393,7 +395,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { .timestamp(timestamp) ); }), - // Ingest Degraded Logs with 27 fields in Apm APP DataSet + // Ingest Degraded Logs with 29 fields in Apm APP DataSet timerange(moment(to).subtract(count, 'minute'), moment(to)) .interval('1m') .rate(1) @@ -413,6 +415,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'trace.id': generateShortId(), test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS], 'cloud.project.id': generateShortId(), + 'event.ingested': new Date().toISOString(), // this works around a geoip limitation in CI tags: CONSISTENT_TAGS, })