Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.x] [Onboarding][Firehose] Switch to the latest CloudFormation template (#204185) #205376

Merged
merged 1 commit into from
Jan 2, 2025
Merged

[8.x] [Onboarding][Firehose] Switch to the latest CloudFormation template (#204185) #205376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions x-pack/solutions/observability/plugins/observability_onboarding/common/aws_firehose.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,11 @@
* 2.0.
*/

export const FIREHOSE_CLOUDFORMATION_STACK_NAME = 'Elastic-CloudwatchLogsAndMetricsToFirehose';
export const FIREHOSE_LOGS_STREAM_NAME = 'Elastic-CloudwatchLogs';
export const FIREHOSE_METRICS_STREAM_NAME = 'Elastic-CloudwatchMetrics';
export const FIREHOSE_CLOUDFORMATION_STACK_NAME = 'Elastic-Firehose';
export const FIREHOSE_STREAM_NAME = 'Elastic-Cloudwatch';

export const FIREHOSE_CLOUDFORMATION_TEMPLATE_URL =
'https://elastic-cloudformation-templates.s3.amazonaws.com/v0.1.0/firehose_default_start.yml';
'https://elastic-cloudformation-templates.s3.amazonaws.com/v0.5.0/firehose_default_start.yml';

export const AWS_INDEX_NAME_LIST = [
// Logs
Expand Down
6 changes: 2 additions & 4 deletions ..._onboarding/public/application/quickstart_flows/firehose/create_stack_command_snippet.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
import React from 'react';
import {
FIREHOSE_CLOUDFORMATION_STACK_NAME,
FIREHOSE_LOGS_STREAM_NAME,
FIREHOSE_METRICS_STREAM_NAME,
FIREHOSE_STREAM_NAME,
} from '../../../../common/aws_firehose';
import { CopyToClipboardButton } from '../shared/copy_to_clipboard_button';
import { DownloadTemplateCallout } from './download_template_callout';
Expand All @@ -42,8 +41,7 @@ export function CreateStackCommandSnippet({
const createStackCommand = buildCreateStackCommand({
templateUrl,
stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
metricsStreamName: FIREHOSE_METRICS_STREAM_NAME,
streamName: FIREHOSE_STREAM_NAME,
encodedApiKey,
elasticsearchUrl,
});
Expand Down
6 changes: 2 additions & 4 deletions ...y_onboarding/public/application/quickstart_flows/firehose/create_stack_in_aws_console.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ import { FormattedMessage } from '@kbn/i18n-react';
import React from 'react';
import {
FIREHOSE_CLOUDFORMATION_STACK_NAME,
FIREHOSE_LOGS_STREAM_NAME,
FIREHOSE_METRICS_STREAM_NAME,
FIREHOSE_STREAM_NAME,
} from '../../../../common/aws_firehose';
import { DownloadTemplateCallout } from './download_template_callout';
import { buildCreateStackAWSConsoleURL } from './utils';
Expand All @@ -33,8 +32,7 @@ export function CreateStackInAWSConsole({
const awsConsoleURL = buildCreateStackAWSConsoleURL({
templateUrl,
stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
metricsStreamName: FIREHOSE_METRICS_STREAM_NAME,
streamName: FIREHOSE_STREAM_NAME,
elasticsearchUrl,
encodedApiKey,
});
Expand Down
4 changes: 2 additions & 2 deletions ...y_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@
import { useFetcher } from '../../../hooks/use_fetcher';
import {
FIREHOSE_CLOUDFORMATION_STACK_NAME,
FIREHOSE_LOGS_STREAM_NAME,
FIREHOSE_STREAM_NAME,
} from '../../../../common/aws_firehose';

export function usePopulatedAWSIndexList() {
return useFetcher((callApi) => {
return callApi('GET /internal/observability_onboarding/firehose/has-data', {
params: {
query: {
logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
streamName: FIREHOSE_STREAM_NAME,
stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
},
},
Expand Down
18 changes: 6 additions & 12 deletions ...ty/plugins/observability_onboarding/public/application/quickstart_flows/firehose/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,13 @@ export const HAS_DATA_FETCH_INTERVAL = 5000;
export function buildCreateStackCommand({
templateUrl,
stackName,
logsStreamName,
metricsStreamName,
streamName,
encodedApiKey,
elasticsearchUrl,
}: {
templateUrl: string;
stackName: string;
logsStreamName: string;
metricsStreamName: string;
streamName: string;
encodedApiKey: string;
elasticsearchUrl: string;
}) {
Expand All @@ -29,8 +27,7 @@ export function buildCreateStackCommand({
aws cloudformation create-stack
--stack-name ${stackName}
--template-url ${escapedTemplateUrl}
--parameters ParameterKey=FirehoseStreamNameForLogs,ParameterValue=${logsStreamName}
ParameterKey=FirehoseStreamNameForMetrics,ParameterValue=${metricsStreamName}
--parameters ParameterKey=FirehoseStreamName,ParameterValue=${streamName}
ParameterKey=ElasticEndpointURL,ParameterValue=${escapedElasticsearchUrl}
ParameterKey=ElasticAPIKey,ParameterValue=${encodedApiKey}
--capabilities CAPABILITY_IAM
Expand All @@ -54,15 +51,13 @@ export function buildStackStatusCommand({ stackName }: { stackName: string }) {
export function buildCreateStackAWSConsoleURL({
templateUrl,
stackName,
logsStreamName,
metricsStreamName,
streamName,
elasticsearchUrl,
encodedApiKey,
}: {
templateUrl: string;
stackName: string;
logsStreamName: string;
metricsStreamName: string;
streamName: string;
elasticsearchUrl: string;
encodedApiKey: string;
}): string {
Expand All @@ -76,8 +71,7 @@ export function buildCreateStackAWSConsoleURL({
* which triggers the eslint rule.
*/
/* eslint-disable @typescript-eslint/naming-convention */
param_FirehoseStreamNameForLogs: logsStreamName,
param_FirehoseStreamNameForMetrics: metricsStreamName,
param_FirehoseStreamName: streamName,
param_ElasticEndpointURL: elasticsearchUrl,
param_ElasticAPIKey: encodedApiKey,
/* eslint-enable @typescript-eslint/naming-convention */
Expand Down
6 changes: 3 additions & 3 deletions .../solutions/observability/plugins/observability_onboarding/server/routes/firehose/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,13 @@ const hasFirehoseDataRoute = createObservabilityOnboardingServerRoute({
endpoint: 'GET /internal/observability_onboarding/firehose/has-data',
params: t.type({
query: t.type({
logsStreamName: t.string,
streamName: t.string,
stackName: t.string,
}),
}),
options: { tags: [] },
async handler(resources): Promise<HasFirehoseDataRouteResponse> {
const { logsStreamName, stackName } = resources.params.query;
const { streamName, stackName } = resources.params.query;
const { elasticsearch } = await resources.context.core;
const indexPatternList = AWS_INDEX_NAME_LIST.map((index) => `${index}-*`);

Expand All @@ -105,7 +105,7 @@ const hasFirehoseDataRoute = createObservabilityOnboardingServerRoute({
query: {
bool: {
should: [
...termQuery('aws.kinesis.name', logsStreamName),
...termQuery('aws.kinesis.name', streamName),
...wildcardQuery('aws.exporter.arn', stackName),
],
},
Expand Down
14 changes: 7 additions & 7 deletions x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import moment from 'moment';
import { FtrProviderContext } from '../../../ftr_provider_context';

const CF_COMMAND_REGEXP =
/aws cloudformation create-stack --stack-name (\S+) --template-url \S+ --parameters ParameterKey=FirehoseStreamNameForLogs,ParameterValue=(\S+) .+? --capabilities CAPABILITY_IAM/;
/aws cloudformation create-stack --stack-name (\S+) --template-url \S+ --parameters ParameterKey=FirehoseStreamName,ParameterValue=(\S+) .+? --capabilities CAPABILITY_IAM/;

export default function ({ getPageObjects, getService }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'svlCommonPage']);
Expand Down Expand Up @@ -60,9 +60,9 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
const AWS_SERVICE_ID = 'vpc-flow';
await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton');
const copiedCommand = await browser.getClipboardValue();
const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
const [, _stackName, streamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];

expect(logsStreamName).toBeDefined();
expect(streamName).toBeDefined();

await browser.execute(`window.dispatchEvent(new Event("blur"))`);

Expand All @@ -75,7 +75,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
.rate(1)
.generator((timestamp) => {
return log.create().dataset(DATASET).timestamp(timestamp).defaults({
'aws.kinesis.name': logsStreamName,
'aws.kinesis.name': streamName,
});
})
);
Expand All @@ -89,11 +89,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
const AWS_SERVICE_ID = 'vpc-flow';
await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton');
const copiedCommand = await browser.getClipboardValue();
const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
const [, _stackName, streamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];

await testSubjects.missingOrFail('observabilityOnboardingFirehosePanelExistingDataCallout');

expect(logsStreamName).toBeDefined();
expect(streamName).toBeDefined();

// Simulate Firehose stream ingesting log files
const to = new Date().toISOString();
Expand All @@ -104,7 +104,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
.rate(1)
.generator((timestamp) => {
return log.create().dataset(DATASET).timestamp(timestamp).defaults({
'aws.kinesis.name': logsStreamName,
'aws.kinesis.name': streamName,
});
})
);
Expand Down
Loading