[8.x] [Observability Onboarding] Show existing data callout in Firehose flow (#203072)

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/existing_data_callout.tsx

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { EuiCallOut, useIsWithinMaxBreakpoint } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+export function ExistingDataCallout() {
+  const isMobile = useIsWithinMaxBreakpoint('s');
+
+  return (
+    <div css={{ maxWidth: isMobile ? '100%' : '80%' }}>
+      <EuiCallOut
+        title={i18n.translate('xpack.observability_onboarding.firehose.existingDataCallout.title', {
+          defaultMessage: 'This workflow has been used before.',
+        })}
+        iconType="iInCircle"
+        color="warning"
+        data-test-subj="observabilityOnboardingFirehosePanelExistingDataCallout"
+      >
+        <p>
+          {i18n.translate(
+            'xpack.observability_onboarding.firehose.existingDataCallout.description',
+            {
+              defaultMessage: `If the Amazon Firehose Data stream(s) associated with this workflow are still active, you will encounter errors during onboarding. Navigate to Step 3 below in order to explore your services.`,
+            }
+          )}
+        </p>
+      </EuiCallOut>
+    </div>
+  );
+}

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/index.tsx

@@ -31,6 +31,8 @@ import { useFirehoseFlow } from './use_firehose_flow';
 import { VisualizeData } from './visualize_data';
 import { ObservabilityOnboardingAppServices } from '../../..';
 import { useWindowBlurDataMonitoringTrigger } from '../shared/use_window_blur_data_monitoring_trigger';
+import { ExistingDataCallout } from './existing_data_callout';
+import { usePopulatedAWSIndexList } from './use_populated_aws_index_list';
 
 const OPTIONS = [
   {
@@ -61,6 +63,9 @@ export function FirehosePanel() {
     },
   } = useKibana<ObservabilityOnboardingAppServices>();
   const { data, status, error, refetch } = useFirehoseFlow();
+  const { data: populatedAWSIndexList } = usePopulatedAWSIndexList();
+
+  const hasExistingData = Array.isArray(populatedAWSIndexList) && populatedAWSIndexList.length > 0;
 
   const telemetryEventContext: OnboardingFlowEventContext = useMemo(
     () => ({
@@ -72,12 +77,13 @@ export function FirehosePanel() {
     [cloudServiceProvider, selectedOptionId]
   );
 
-  const isMonitoringData = useWindowBlurDataMonitoringTrigger({
-    isActive: status === FETCH_STATUS.SUCCESS,
-    onboardingFlowType: 'firehose',
-    onboardingId: data?.onboardingId,
-    telemetryEventContext,
-  });
+  const isMonitoringData =
+    useWindowBlurDataMonitoringTrigger({
+      isActive: status === FETCH_STATUS.SUCCESS,
+      onboardingFlowType: 'firehose',
+      onboardingId: data?.onboardingId,
+      telemetryEventContext,
+    }) || hasExistingData;
 
   const onOptionChange = useCallback((id: string) => {
     setSelectedOptionId(id as CreateStackOption);
@@ -193,13 +199,20 @@ export function FirehosePanel() {
         <VisualizeData
           selectedCreateStackOption={selectedOptionId}
           onboardingId={data.onboardingId}
+          hasExistingData={hasExistingData}
         />
       ),
     },
   ];
 
   return (
     <EuiPanel hasBorder paddingSize="xl">
+      {hasExistingData && (
+        <>
+          <ExistingDataCallout />
+          <EuiSpacer size="xl" />
+        </>
+      )}
       <EuiSteps steps={steps} />
       <FeedbackButtons flow="firehose" />
     </EuiPanel>

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/use_populated_aws_index_list.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useFetcher } from '../../../hooks/use_fetcher';
+import {
+  FIREHOSE_CLOUDFORMATION_STACK_NAME,
+  FIREHOSE_LOGS_STREAM_NAME,
+} from '../../../../common/aws_firehose';
+
+export function usePopulatedAWSIndexList() {
+  return useFetcher((callApi) => {
+    return callApi('GET /internal/observability_onboarding/firehose/has-data', {
+      params: {
+        query: {
+          logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
+          stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
+        },
+      },
+    });
+  }, []);
+}

x-pack/plugins/observability_solution/observability_onboarding/public/application/quickstart_flows/firehose/visualize_data.tsx

@@ -13,11 +13,7 @@ import { unionBy } from 'lodash';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { ObservabilityOnboardingAppServices } from '../../..';
-import {
-  FIREHOSE_CLOUDFORMATION_STACK_NAME,
-  FIREHOSE_LOGS_STREAM_NAME,
-} from '../../../../common/aws_firehose';
-import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
+import { FETCH_STATUS } from '../../../hooks/use_fetcher';
 import { AccordionWithIcon } from '../shared/accordion_with_icon';
 import { GetStartedPanel } from '../shared/get_started_panel';
 import {
@@ -28,34 +24,25 @@ import { AutoRefreshCallout } from './auto_refresh_callout';
 import { ProgressCallout } from './progress_callout';
 import { HAS_DATA_FETCH_INTERVAL } from './utils';
 import { CreateStackOption } from './types';
+import { usePopulatedAWSIndexList } from './use_populated_aws_index_list';
 
 const REQUEST_PENDING_STATUS_LIST = [FETCH_STATUS.LOADING, FETCH_STATUS.NOT_INITIATED];
 
 interface Props {
   onboardingId: string;
   selectedCreateStackOption: CreateStackOption;
+  hasExistingData: boolean;
 }
 
-export function VisualizeData({ onboardingId, selectedCreateStackOption }: Props) {
+export function VisualizeData({ onboardingId, selectedCreateStackOption, hasExistingData }: Props) {
   const accordionId = useGeneratedHtmlId({ prefix: 'accordion' });
   const [orderedVisibleAWSServiceList, setOrderedVisibleAWSServiceList] = useState<
     AWSServiceGetStartedConfig[]
   >([]);
-  const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState<boolean>(true);
-  const {
-    data: populatedAWSIndexList,
-    status,
-    refetch,
-  } = useFetcher((callApi) => {
-    return callApi('GET /internal/observability_onboarding/firehose/has-data', {
-      params: {
-        query: {
-          logsStreamName: FIREHOSE_LOGS_STREAM_NAME,
-          stackName: FIREHOSE_CLOUDFORMATION_STACK_NAME,
-        },
-      },
-    });
-  }, []);
+  const [shouldShowDataReceivedToast, setShouldShowDataReceivedToast] = useState<boolean>(
+    !hasExistingData
+  );
+  const { data: populatedAWSIndexList, status, refetch } = usePopulatedAWSIndexList();
   const {
     services: {
       notifications,

x-pack/test_serverless/functional/test_suites/observability/onboarding/firehose.ts

@@ -83,5 +83,39 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       // Checking that an AWS service item is visible after data is detected
       await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`);
     });
+
+    it('shows the existing data callout and detected AWS services when data was ingested previously', async () => {
+      const DATASET = 'aws.vpcflow';
+      const AWS_SERVICE_ID = 'vpc-flow';
+      await testSubjects.clickWhenNotDisabled('observabilityOnboardingCopyToClipboardButton');
+      const copiedCommand = await browser.getClipboardValue();
+      const [, _stackName, logsStreamName] = copiedCommand.match(CF_COMMAND_REGEXP) ?? [];
+
+      await testSubjects.missingOrFail('observabilityOnboardingFirehosePanelExistingDataCallout');
+
+      expect(logsStreamName).toBeDefined();
+
+      // Simulate Firehose stream ingesting log files
+      const to = new Date().toISOString();
+      const count = 1;
+      await synthtrace.index(
+        timerange(moment(to).subtract(count, 'minute'), moment(to))
+          .interval('1m')
+          .rate(1)
+          .generator((timestamp) => {
+            return log.create().dataset(DATASET).timestamp(timestamp).defaults({
+              'aws.kinesis.name': logsStreamName,
+            });
+          })
+      );
+
+      await browser.refresh();
+
+      // Checking that the existing data callout is visible after data is detected
+      await testSubjects.isDisplayed('observabilityOnboardingFirehosePanelExistingDataCallout');
+
+      // Checking that an AWS service item is visible after data is detected
+      await testSubjects.isDisplayed(`observabilityOnboardingAWSService-${AWS_SERVICE_ID}`);
+    });
   });
 }