Trace Analytics support for custom sources

common/constants/trace_analytics.ts

@@ -7,14 +7,20 @@ export const JAEGER_INDEX_NAME = '*jaeger-span-*';
 export const JAEGER_SERVICE_INDEX_NAME = '*jaeger-service*';
 export const DATA_PREPPER_INDEX_NAME = 'otel-v1-apm-span-*';
 export const DATA_PREPPER_SERVICE_INDEX_NAME = 'otel-v1-apm-service-map*';
-export const TRACE_ANALYTICS_DATE_FORMAT = 'MM/DD/YYYY HH:mm:ss';
-export const TRACE_ANALYTICS_PLOTS_DATE_FORMAT = 'MMM D, YYYY HH:mm:ss';
+export const TRACE_ANALYTICS_DATE_FORMAT = 'MM/DD/YYYY HH:mm:ss.SSS';
+export const TRACE_ANALYTICS_PLOTS_DATE_FORMAT = 'MMM D, YYYY HH:mm:ss.SSS';
 export const SERVICE_MAP_MAX_NODES = 500;
 // size limit when requesting edge related queries, not necessarily the number of edges
 export const SERVICE_MAP_MAX_EDGES = 1000;
 export const TRACES_MAX_NUM = 3000;
-export const TRACE_ANALYTICS_DOCUMENTATION_LINK = 'https://opensearch.org/docs/latest/observability-plugin/trace/index/';
+export const TRACE_ANALYTICS_DOCUMENTATION_LINK =
+  'https://opensearch.org/docs/latest/observability-plugin/trace/index/';
 
-export const TRACE_ANALYTICS_JAEGER_INDICES_ROUTE = '/api/observability/trace_analytics/jaeger_indices';
-export const TRACE_ANALYTICS_DATA_PREPPER_INDICES_ROUTE = '/api/observability/trace_analytics/data_prepper_indices';
+export const TRACE_ANALYTICS_JAEGER_INDICES_ROUTE =
+  '/api/observability/trace_analytics/jaeger_indices';
+export const TRACE_ANALYTICS_DATA_PREPPER_INDICES_ROUTE =
+  '/api/observability/trace_analytics/data_prepper_indices';
 export const TRACE_ANALYTICS_DSL_ROUTE = '/api/observability/trace_analytics/query';
+
+export const TRACE_CUSTOM_SPAN_INDEX_SETTING = 'observability:traceAnalyticsSpanIndices';
+export const TRACE_CUSTOM_SERVICE_INDEX_SETTING = 'observability:traceAnalyticsServiceIndices';

common/types/trace_analytics.ts

@@ -52,3 +52,5 @@ export interface GraphVisEdge {
   to: number;
   color: string;
 }
+
+export type TraceAnalyticsMode = 'jaeger' | 'data_prepper' | 'custom_data_prepper';

public/components/application_analytics/components/application.tsx

@@ -134,7 +134,7 @@
   const [serviceFlyoutName, setServiceFlyoutName] = useState<string>('');
   const [traceFlyoutId, setTraceFlyoutId] = useState<string>('');
   const [spanFlyoutId, setSpanFlyoutId] = useState<string>('');
   const [spanDSL, setSpanDSL] = useState<any>({});
   const [totalSpans, setTotalSpans] = useState<number>(0);
   const [editVizId, setEditVizId] = useState<string>('');
   const [visWithAvailability, setVisWithAvailability] = useState<EuiSelectOption[]>([]);
@@ -157,7 +157,7 @@
     sessionStorage.setItem(`${application.name}EndTime`, newEndTime);
   };
 
   const addSpanFilter = (field: string, value: any) => {
     const newFilters = [...filters];
     const index = newFilters.findIndex(({ field: filterField }) => field === filterField);
     if (index === -1) {
@@ -286,7 +286,7 @@
     },
   ];
 
   const nameColumnAction = (item: any) => openServiceFlyout(item);
   const traceColumnAction = () => switchToTrace();
 
   const getService = () => {
@@ -314,8 +314,6 @@
     setSelectedTab(TAB_TRACE_ID);
   };
 
-  const traceIdColumnAction = (item: any) => openTraceFlyout(item);
-
   const getTrace = () => {
     return (
       <>
@@ -325,7 +323,7 @@
           page="app"
           parentBreadcrumb={parentBreadcrumbs[0]}
           childBreadcrumbs={childBreadcrumbs}
-          traceIdColumnAction={traceIdColumnAction}
+          openTraceFlyout={openTraceFlyout}
           startTime={appStartTime}
           endTime={appEndTime}
           setStartTime={setStartTimeForApp}

public/components/application_analytics/components/flyout_components/service_detail_flyout.tsx

@@ -49,10 +49,10 @@
     openSpanFlyout,
     mode,
   } = props;
   const [fields, setFields] = useState<any>({});
   const [serviceMap, setServiceMap] = useState<ServiceObject>({});
   const [total, setTotal] = useState(0);
   const [DSL, setDSL] = useState<any>({});
   const [serviceMapIdSelected, setServiceMapIdSelected] = useState<
     'latency' | 'error_rate' | 'throughput'
   >('latency');
@@ -133,7 +133,7 @@
     handleServiceViewRequest(serviceName, http, serviceDSL, setFields, mode);
     handleServiceMapRequest(http, serviceDSL, mode, '', setServiceMap, serviceName);
     const spanDSL = filtersToDsl(mode, filters, query, startTime, endTime, 'app', appConfigs);
-    spanDSL.query.bool.must.push({
+    spanDSL.query.bool.filter.push({
       term: {
         serviceName,
       },

public/components/application_analytics/components/flyout_components/trace_detail_render.tsx

@@ -3,28 +3,33 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { EuiText, EuiSpacer, EuiHorizontalRule, EuiCodeBlock } from '@elastic/eui';
+import { EuiCodeBlock, EuiHorizontalRule, EuiSpacer, EuiText } from '@elastic/eui';
 import React, { useEffect, useMemo, useState } from 'react';
+import { HttpStart } from '../../../../../../../src/core/public';
+import { TraceAnalyticsMode } from '../../../../../common/types/trace_analytics';
 import { ServiceBreakdownPanel } from '../../../trace_analytics/components/traces/service_breakdown_panel';
 import { SpanDetailPanel } from '../../../trace_analytics/components/traces/span_detail_panel';
 import {
-  handleTraceViewRequest,
-  handleServicesPieChartRequest,
   handlePayloadRequest,
+  handleServicesPieChartRequest,
+  handleTraceViewRequest,
 } from '../../../trace_analytics/requests/traces_request_handler';
-import { HttpStart } from '../../../../../../../src/core/public';
 import { getListItem } from '../../helpers/utils';
-import { TraceAnalyticsMode } from '../../../../../public/components/trace_analytics/home';
 
 interface TraceDetailRenderProps {
   traceId: string;
   http: HttpStart;
   openSpanFlyout: (spanId: string) => void;
-  mode : TraceAnalyticsMode
+  mode: TraceAnalyticsMode;
 }
 
-export const TraceDetailRender = ({ traceId, http, openSpanFlyout, mode }: TraceDetailRenderProps) => {
+export const TraceDetailRender = ({
+  traceId,
+  http,
+  openSpanFlyout,
+  mode,
+}: TraceDetailRenderProps) => {
   const [fields, setFields] = useState<any>({});
   const [serviceBreakdownData, setServiceBreakdownData] = useState([]);
   const [payloadData, setPayloadData] = useState('');
   const [colorMap, setColorMap] = useState({});
@@ -83,13 +88,13 @@
         ) : null}
       </>
     );
   }, [traceId, fields, serviceBreakdownData, colorMap, payloadData]);
 
   useEffect(() => {
     handleTraceViewRequest(traceId, http, fields, setFields, mode);
     handleServicesPieChartRequest(traceId, http, setServiceBreakdownData, setColorMap, mode);
     handlePayloadRequest(traceId, http, payloadData, setPayloadData, mode);
   }, [traceId]);
 
   return renderContent;
 };

public/components/trace_analytics/components/common/__tests__/helper_functions.test.tsx

@@ -5,7 +5,6 @@
 
 import { configure, mount, shallow } from 'enzyme';
 import Adapter from 'enzyme-adapter-react-16';
-import { TraceAnalyticsMode } from 'public/components/trace_analytics/home';
 import React from 'react';
 import { TEST_SERVICE_MAP, TEST_SERVICE_MAP_GRAPH } from '../../../../../../test/constants';
 import {
@@ -38,7 +37,9 @@ describe('Helper functions', () => {
 
   it('renders no match and missing configuration messages', () => {
     const noMatchMessage = shallow(<NoMatchMessage size="s" />);
-    const missingConfigurationMessage = shallow(<MissingConfigurationMessage mode='data_prepper'/>)
+    const missingConfigurationMessage = shallow(
+      <MissingConfigurationMessage mode="data_prepper" />
+    );
     expect(noMatchMessage).toMatchSnapshot();
     expect(missingConfigurationMessage).toMatchSnapshot();
   });
@@ -137,16 +138,16 @@ describe('Helper functions', () => {
       );
     const existsDSL = getTestDslFromFilters();
     expect(JSON.stringify(existsDSL)).toEqual(
-      '{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}},{"exists":{"field":"traceGroup"}}],"filter":[],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
+      '{"query":{"bool":{"must":[{"exists":{"field":"traceGroup"}}],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
     );
 
     const isDSL = getTestDslFromFilters('traceGroup', 'is');
     expect(JSON.stringify(isDSL)).toEqual(
-      '{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}},{"term":{"traceGroup":{"from":"100","to":"∞"}}}],"filter":[],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
+      '{"query":{"bool":{"must":[{"term":{"traceGroup":{"from":"100","to":"∞"}}}],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
     );
     const isBetweenDSL = getTestDslFromFilters('durationInNanos', 'is between');
     expect(JSON.stringify(isBetweenDSL)).toEqual(
-      `{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}},{"range":{"durationInNanos":{"gte":"100"}}}],"filter":[],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}`
+      '{"query":{"bool":{"must":[{"range":{"durationInNanos":{"gte":"100"}}}],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
     );
 
     const customDSL = filtersToDsl(
@@ -166,7 +167,7 @@ describe('Helper functions', () => {
       'now'
     );
     expect(JSON.stringify(customDSL)).toEqual(
-      `{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"filter":[],"should":["test"],"must_not":[],"minimum_should_match":1}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":["test"],"minimum_should_match":1}}}}}`
+      '{"query":{"bool":{"must":[],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":["test"],"must_not":[],"minimum_should_match":1}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":["test"],"minimum_should_match":1}}}}}'
     );
   });
 });

public/components/trace_analytics/components/common/custom_index_flyout.tsx

@@ -0,0 +1,167 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  EuiCallOut,
+  EuiCompressedFieldText,
+  EuiDescribedFormGroup,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiFlyout,
+  EuiFlyoutBody,
+  EuiFlyoutFooter,
+  EuiFlyoutHeader,
+  EuiFormRow,
+  EuiLink,
+  EuiSmallButton,
+  EuiSmallButtonEmpty,
+  EuiTitle,
+} from '@elastic/eui';
+import React, { Fragment, useEffect, useState } from 'react';
+import {
+  TRACE_CUSTOM_SERVICE_INDEX_SETTING,
+  TRACE_CUSTOM_SPAN_INDEX_SETTING,
+} from '../../../../../common/constants/trace_analytics';
+import { uiSettingsService } from '../../../../../common/utils';
+import { useToast } from '../../../common/toast';
+
+interface CustomIndexFlyoutProps {
+  isFlyoutVisible: boolean;
+  setIsFlyoutVisible: React.Dispatch<React.SetStateAction<boolean>>;
+}
+
+export const CustomIndexFlyout = ({
+  isFlyoutVisible,
+  setIsFlyoutVisible,
+}: CustomIndexFlyoutProps) => {
+  const { setToast } = useToast();
+  const [spanIndices, setSpanIndices] = useState('');
+  const [serviceIndices, setServiceIndices] = useState('');
+  const [isLoading, setIsLoading] = useState(false);
+
+  const onChangeSpanIndices = (e: { target: { value: React.SetStateAction<string> } }) => {
+    setSpanIndices(e.target.value);
+  };
+
+  const onChangeServiceIndices = (e: { target: { value: React.SetStateAction<string> } }) => {
+    setServiceIndices(e.target.value);
+  };
+
+  useEffect(() => {
+    setSpanIndices(uiSettingsService.get(TRACE_CUSTOM_SPAN_INDEX_SETTING));
+    setServiceIndices(uiSettingsService.get(TRACE_CUSTOM_SERVICE_INDEX_SETTING));
+  }, [uiSettingsService]);
+
+  const onSaveIndices = async () => {
+    try {
+      setIsLoading(true);
+      await uiSettingsService.set(TRACE_CUSTOM_SPAN_INDEX_SETTING, spanIndices);
+      await uiSettingsService.set(TRACE_CUSTOM_SERVICE_INDEX_SETTING, serviceIndices);
+      setIsLoading(false);
+      setToast('Updated trace analytics sources successfully', 'success');
+    } catch (error) {
+      console.error(error);
+      setToast('Failed to update trace analytics sources', 'danger');
+    }
+    setIsLoading(false);
+  };
+
+  const callout = (
+    <EuiCallOut
+      title="Custom indices in trace analytics is an experimental feature"
+      color="warning"
+      iconType="help"
+    >
+      <p>
+        This feature is experimental, all indices added here should adhere to data prepper index
+        mappings. For more information on mappings, visit{' '}
+        <EuiLink
+          href="https://github.com/opensearch-project/data-prepper/tree/main/docs/schemas/trace-analytics"
+          target="_blank"
+        >
+          schema documentation
+        </EuiLink>
+        .
+      </p>
+    </EuiCallOut>
+  );
+  let flyout;
+
+  if (isFlyoutVisible) {
+    flyout = (
+      <EuiFlyout ownFocus onClose={() => setIsFlyoutVisible(false)} aria-labelledby="flyoutTitle">
+        <EuiFlyoutHeader hasBorder>
+          <EuiTitle size="m">
+            <h3 id="flyoutTitle">Manage custom trace source</h3>
+          </EuiTitle>
+        </EuiFlyoutHeader>
+        <EuiFlyoutBody banner={callout}>
+          <EuiDescribedFormGroup
+            title={<h3>Custom span indices</h3>}
+            description={
+              <Fragment>
+                Configure custom span indices to be used by the trace analytics plugin
+              </Fragment>
+            }
+          >
+            <EuiFormRow label="Custom span indices">
+              <EuiCompressedFieldText
+                name="spanIndices"
+                aria-label="spanIndices"
+                placeholder="index1,cluster1:index2,cluster:index3"
+                value={spanIndices}
+                onChange={onChangeSpanIndices}
+              />
+            </EuiFormRow>
+          </EuiDescribedFormGroup>
+          <EuiDescribedFormGroup
+            title={<h3>Custom service indices</h3>}
+            description={
+              <Fragment>
+                Configure custom service indices to be used by the trace analytics plugin
+              </Fragment>
+            }
+          >
+            <EuiFormRow label="Custom service indices">
+              <EuiCompressedFieldText
+                name="serviceIndices"
+                aria-label="serviceIndices"
+                placeholder="index1,cluster1:index2,cluster:index3"
+                value={serviceIndices}
+                onChange={onChangeServiceIndices}
+              />
+            </EuiFormRow>
+          </EuiDescribedFormGroup>
+        </EuiFlyoutBody>
+        <EuiFlyoutFooter>
+          <EuiFlexGroup justifyContent="spaceBetween">
+            <EuiFlexItem grow={false}>
+              <EuiSmallButtonEmpty
+                iconType="cross"
+                onClick={() => setIsFlyoutVisible(false)}
+                flush="left"
+              >
+                Close
+              </EuiSmallButtonEmpty>
+            </EuiFlexItem>
+            <EuiFlexItem grow={false}>
+              <EuiSmallButton
+                onClick={async () => {
+                  await onSaveIndices();
+                  setIsFlyoutVisible(false);
+                }}
+                fill
+                isLoading={isLoading}
+              >
+                Save
+              </EuiSmallButton>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFlyoutFooter>
+      </EuiFlyout>
+    );
+  }
+  return <div>{flyout}</div>;
+};

public/components/trace_analytics/components/common/filters/filter_helpers.tsx

@@ -6,20 +6,20 @@
 import {
   EuiCompressedComboBox,
   EuiCompressedFieldText,
-  EuiFormControlLayoutDelimited,
   EuiCompressedFormRow,
+  EuiFormControlLayoutDelimited,
   EuiSpacer,
 } from '@elastic/eui';
 import get from 'lodash/get';
-import { TraceAnalyticsMode } from 'public/components/trace_analytics/home';
 import React from 'react';
+import { TraceAnalyticsMode } from '../../../../../../common/types/trace_analytics';
 
 const getFields = (
   mode: TraceAnalyticsMode,
   page: 'dashboard' | 'traces' | 'services' | 'app',
   attributesFilterFields: string[]
 ) =>
-  mode === 'data_prepper'
+  mode === 'data_prepper' || mode === 'custom_data_prepper'
     ? {
         dashboard: ['traceGroup', 'serviceName', 'error', 'status.message', 'latency'],
         traces: [
@@ -151,7 +151,7 @@
 export const getValueComponent = (
   field: string,
   operator: string,
   value: any,
   setValue: (v: any) => void
 ) => {
   const textField = (

public/components/trace_analytics/components/common/filters/filters.tsx

@@ -6,18 +6,18 @@
 import {
   EuiBadge,
   EuiButtonEmpty,
-  EuiSmallButtonIcon,
   EuiContextMenu,
   EuiContextMenuPanelDescriptor,
   EuiFlexGroup,
   EuiFlexItem,
   EuiIcon,
   EuiPopover,
   EuiPopoverTitle,
+  EuiSmallButtonIcon,
   EuiTextColor,
 } from '@elastic/eui';
-import { TraceAnalyticsMode } from 'public/components/trace_analytics/home';
 import React, { useMemo, useState } from 'react';
+import { TraceAnalyticsMode } from '../../../../../../common/types/trace_analytics';
 import { FilterEditPopover } from './filter_edit_popover';
 import { getFilterFields, getValidFilterFields } from './filter_helpers';