From a64622fcadd8982ac7cbc0dc7cba042d60748117 Mon Sep 17 00:00:00 2001 From: Mykola Harmash Date: Thu, 2 Jan 2025 12:39:58 +0100 Subject: [PATCH] [Onboarding][Firehose] Switch to the latest CloudFormation template (#204185) Closes https://github.com/elastic/kibana/issues/203563 Updates the CF template to the latest available version and adjust the code to use a single Firehose stream created by the new template. You can use `Elastic Observability` AWS account (available through Okta) and the latest Kibana deployment from this PR to test the flow. ![CleanShot 2024-12-17 at 11 42 33@2x](https://github.com/user-attachments/assets/ac9ba9eb-1c9f-48fb-9fca-ed518970ac9b) (cherry picked from commit db083d17fc4602e5c7849eb2da9b304b3067f563) --- .../common/aws_firehose.ts | 7 +++---- .../firehose/create_stack_command_snippet.tsx | 6 ++---- .../firehose/create_stack_in_aws_console.tsx | 6 ++---- .../firehose/use_populated_aws_index_list.ts | 4 ++-- .../quickstart_flows/firehose/utils.ts | 18 ++++++------------ .../server/routes/firehose/route.ts | 6 +++--- .../observability/onboarding/firehose.ts | 14 +++++++------- 7 files changed, 25 insertions(+), 36 deletions(-) diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_firehose.ts b/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_firehose.ts index 2fa8ae220bdb3..908aa2c144fb3 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_firehose.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/common/aws_firehose.ts @@ -5,12 +5,11 @@ * 2.0. */ -export const FIREHOSE_CLOUDFORMATION_STACK_NAME = 'Elastic-CloudwatchLogsAndMetricsToFirehose'; -export const FIREHOSE_LOGS_STREAM_NAME = 'Elastic-CloudwatchLogs'; -export const FIREHOSE_METRICS_STREAM_NAME = 'Elastic-CloudwatchMetrics'; +export const FIREHOSE_CLOUDFORMATION_STACK_NAME = 'Elastic-Firehose'; +export const FIREHOSE_STREAM_NAME = 'Elastic-Cloudwatch'; export const FIREHOSE_CLOUDFORMATION_TEMPLATE_URL = - 'https://elastic-cloudformation-templates.s3.amazonaws.com/v0.1.0/firehose_default_start.yml'; + 'https://elastic-cloudformation-templates.s3.amazonaws.com/v0.5.0/firehose_default_start.yml'; export const AWS_INDEX_NAME_LIST = [ // Logs diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx index dedc05c701e00..963e5a21479d8 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx @@ -18,8 +18,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import React from 'react'; import { FIREHOSE_CLOUDFORMATION_STACK_NAME, - FIREHOSE_LOGS_STREAM_NAME, - FIREHOSE_METRICS_STREAM_NAME, + FIREHOSE_STREAM_NAME, } from '../../../../common/aws_firehose'; import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button'; import { DownloadTemplateCallout } from './download_template_callout'; @@ -42,8 +41,7 @@ export function CreateStackCommandSnippet({ const createStackCommand = buildCreateStackCommand({ templateUrl, stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME, - logsStreamName: FIREHOSE_LOGS_STREAM_NAME, - metricsStreamName: FIREHOSE_METRICS_STREAM_NAME, + streamName: FIREHOSE_STREAM_NAME, encodedApiKey, elasticsearchUrl, }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx index 3c62632b382ad..41b08718e6fe9 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx @@ -11,8 +11,7 @@ import { FormattedMessage } from '@kbn/i18n-react'; import React from 'react'; import { FIREHOSE_CLOUDFORMATION_STACK_NAME, - FIREHOSE_LOGS_STREAM_NAME, - FIREHOSE_METRICS_STREAM_NAME, + FIREHOSE_STREAM_NAME, } from '../../../../common/aws_firehose'; import { DownloadTemplateCallout } from './download_template_callout'; import { buildCreateStackAWSConsoleURL } from './utils'; @@ -33,8 +32,7 @@ export function CreateStackInAWSConsole({ const awsConsoleURL = buildCreateStackAWSConsoleURL({ templateUrl, stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME, - logsStreamName: FIREHOSE_LOGS_STREAM_NAME, - metricsStreamName: FIREHOSE_METRICS_STREAM_NAME, + streamName: FIREHOSE_STREAM_NAME, elasticsearchUrl, encodedApiKey, }); diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts index 070a5aed4e2db..c2f048ceae124 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts @@ -8,7 +8,7 @@ import { useFetcher } from '../../../hooks/use_fetcher'; import { FIREHOSE_CLOUDFORMATION_STACK_NAME, - FIREHOSE_LOGS_STREAM_NAME, + FIREHOSE_STREAM_NAME, } from '../../../../common/aws_firehose'; export function usePopulatedAWSIndexList() { @@ -16,7 +16,7 @@ export function usePopulatedAWSIndexList() { return callApi('GET /internal/observability_onboarding/firehose/has-data', { params: { query: { - logsStreamName: FIREHOSE_LOGS_STREAM_NAME, + streamName: FIREHOSE_STREAM_NAME, stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME, }, }, diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts index 0fa28276f7fcc..96b81522196ea 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts @@ -10,15 +10,13 @@ export const HAS_DATA_FETCH_INTERVAL = 5000; export function buildCreateStackCommand({ templateUrl, stackName, - logsStreamName, - metricsStreamName, + streamName, encodedApiKey, elasticsearchUrl, }: { templateUrl: string; stackName: string; - logsStreamName: string; - metricsStreamName: string; + streamName: string; encodedApiKey: string; elasticsearchUrl: string; }) { @@ -29,8 +27,7 @@ export function buildCreateStackCommand({ aws cloudformation create-stack --stack-name ${stackName} --template-url ${escapedTemplateUrl} - --parameters ParameterKey=FirehoseStreamNameForLogs,ParameterValue=${logsStreamName} - ParameterKey=FirehoseStreamNameForMetrics,ParameterValue=${metricsStreamName} + --parameters ParameterKey=FirehoseStreamName,ParameterValue=${streamName} ParameterKey=ElasticEndpointURL,ParameterValue=${escapedElasticsearchUrl} ParameterKey=ElasticAPIKey,ParameterValue=${encodedApiKey} --capabilities CAPABILITY_IAM @@ -54,15 +51,13 @@ export function buildStackStatusCommand({ stackName }: { stackName: string }) { export function buildCreateStackAWSConsoleURL({ templateUrl, stackName, - logsStreamName, - metricsStreamName, + streamName, elasticsearchUrl, encodedApiKey, }: { templateUrl: string; stackName: string; - logsStreamName: string; - metricsStreamName: string; + streamName: string; elasticsearchUrl: string; encodedApiKey: string; }): string { @@ -76,8 +71,7 @@ export function buildCreateStackAWSConsoleURL({ * which triggers the eslint rule. */ /* eslint-disable @typescript-eslint/naming-convention */ - param_FirehoseStreamNameForLogs: logsStreamName, - param_FirehoseStreamNameForMetrics: metricsStreamName, + param_FirehoseStreamName: streamName, param_ElasticEndpointURL: elasticsearchUrl, param_ElasticAPIKey: encodedApiKey, /* eslint-enable @typescript-eslint/naming-convention */ diff --git a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/firehose/route.ts b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/firehose/route.ts index 64f9a15049fca..a74d9cc691787 100644 --- a/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/firehose/route.ts +++ b/x-pack/solutions/observability/plugins/observability_onboarding/server/routes/firehose/route.ts @@ -81,13 +81,13 @@ const hasFirehoseDataRoute = createObservabilityOnboardingServerRoute({ endpoint: 'GET /internal/observability_onboarding/firehose/has-data', params: t.type({ query: t.type({ - logsStreamName: t.string, + streamName: t.string, stackName: t.string, }), }), options: { tags: [] }, async handler(resources): Promise { - const { logsStreamName, stackName } = resources.params.query; + const { streamName, stackName } = resources.params.query; const { elasticsearch } = await resources.context.core; const indexPatternList = AWS_INDEX_NAME_LIST.map((index) => `${index}-*`); @@ -105,7 +105,7 @@ const hasFirehoseDataRoute = createObservabilityOnboardingServerRoute({ query: { bool: { should: [ - ...termQuery('aws.kinesis.name', logsStreamName), + ...termQuery('aws.kinesis.name', streamName), ...wildcardQuery('aws.exporter.arn', stackName), ], }, diff --git a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts index 163ff88c8ee5d..3421db3a862de 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts @@ -11,7 +11,7 @@ import moment from 'moment'; import { FtrProviderContext } from '../../../ftr_provider_context'; const CF_COMMAND_REGEXP = - /aws cloudformation create-stack --stack-name (\S+) --template-url \S+ --parameters ParameterKey=FirehoseStreamNameForLogs,ParameterValue=(\S+) .+? --capabilities CAPABILITY_IAM/; + /aws cloudformation create-stack --stack-name (\S+) --template-url \S+ --parameters ParameterKey=FirehoseStreamName,ParameterValue=(\S+) .+? --capabilities CAPABILITY_IAM/; export default function ({ getPageObjects, getService }: FtrProviderContext) { const PageObjects = getPageObjects(['common', 'svlCommonPage']); @@ -60,9 +60,9 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const AWS_SERVICE_ID = 'vpc-flow'; await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton'); const copiedCommand = await browser.getClipboardValue(); - const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? []; + const [, _stackName, streamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? []; - expect(logsStreamName).toBeDefined(); + expect(streamName).toBeDefined(); await browser.execute(`window.dispatchEvent(new Event("blur"))`); @@ -75,7 +75,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { .rate(1) .generator((timestamp) => { return log.create().dataset(DATASET).timestamp(timestamp).defaults({ - 'aws.kinesis.name': logsStreamName, + 'aws.kinesis.name': streamName, }); }) ); @@ -89,11 +89,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const AWS_SERVICE_ID = 'vpc-flow'; await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton'); const copiedCommand = await browser.getClipboardValue(); - const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? []; + const [, _stackName, streamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? []; await testSubjects.missingOrFail('observabilityOnboardingFirehosePanelExistingDataCallout'); - expect(logsStreamName).toBeDefined(); + expect(streamName).toBeDefined(); // Simulate Firehose stream ingesting log files const to = new Date().toISOString(); @@ -104,7 +104,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { .rate(1) .generator((timestamp) => { return log.create().dataset(DATASET).timestamp(timestamp).defaults({ - 'aws.kinesis.name': logsStreamName, + 'aws.kinesis.name': streamName, }); }) );