Schema Editor UI

x-pack/plugins/streams/server/routes/index.ts

@@ -14,6 +14,7 @@ import { forkStreamsRoute } from './streams/fork';
 import { listStreamsRoute } from './streams/list';
 import { readStreamRoute } from './streams/read';
 import { resyncStreamsRoute } from './streams/resync';
+import { unmappedFieldsRoute } from './streams/schema/unmapped_fields';
 import { streamsStatusRoutes } from './streams/settings';
 
 export const streamsRouteRepository = {
@@ -27,6 +28,7 @@ export const streamsRouteRepository = {
   ...streamsStatusRoutes,
   ...esqlRoutes,
   ...disableStreamsRoute,
+  ...unmappedFieldsRoute,
 };
 
 export type StreamsRouteRepository = typeof streamsRouteRepository;

x-pack/plugins/streams/server/routes/streams/schema/unmapped_fields.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { z } from '@kbn/zod';
+import { internal, notFound } from '@hapi/boom';
+import { DefinitionNotFound } from '../../../lib/streams/errors';
+import { readStream } from '../../../lib/streams/stream_crud';
+import { createServerRoute } from '../../create_server_route';
+
+const SAMPLE_SIZE = 500;
+
+export const unmappedFieldsRoute = createServerRoute({
+  endpoint: 'GET /api/streams/{id}/schema/unmapped_fields',
+  options: {
+    access: 'internal',
+  },
+  security: {
+    authz: {
+      enabled: false,
+      reason:
+        'This API delegates security to the currently logged in user and their Elasticsearch permissions.',
+    },
+  },
+  params: z.object({
+    path: z.object({ id: z.string() }),
+  }),
+  handler: async ({
+    response,
+    params,
+    request,
+    logger,
+    getScopedClients,
+  }): Promise<{ unmappedFields: string[] }> => {
+    try {
+      const { scopedClusterClient } = await getScopedClients({ request });
+
+      const streamEntity = await readStream({
+        scopedClusterClient,
+        id: params.path.id,
+      });
+
+      const searchBody = {
+        sort: [
+          {
+            '@timestamp': {
+              order: 'desc',
+            },
+          },
+        ],
+        size: SAMPLE_SIZE,
+      };
+
+      const results = await scopedClusterClient.asCurrentUser.search({
+        index: params.path.id,
+        ...searchBody,
+      });
+
+      const sourceFields = new Set<string>();
+
+      results.hits.hits.forEach((hit) => {
+        Object.keys(hit._source as Record<string, unknown>).forEach((field) => {
+          if (!sourceFields.has(field)) {
+            sourceFields.add(field);
+          }
+        });
+      });
+
+      const mappedFields = streamEntity.definition.fields.map((field) => field.name);
+
+      const unmappedFields = Array.from(sourceFields)
+        .filter((field) => !mappedFields.includes(field))
+        .sort();
+
+      return { unmappedFields };
+    } catch (e) {
+      if (e instanceof DefinitionNotFound) {
+        throw notFound(e);
+      }
+
+      throw internal(e);
+    }
+  },
+});

x-pack/plugins/streams_app/public/components/stream_detail_management/index.tsx

@@ -6,8 +6,8 @@
  */
 import React from 'react';
 import { i18n } from '@kbn/i18n';
-import { StreamDefinition } from '@kbn/streams-plugin/common';
 import { EuiButtonGroup, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
+import type { APIReturnType } from '@kbn/streams-plugin/public/api';
 import { useStreamsAppParams } from '../../hooks/use_streams_app_params';
 import { RedirectTo } from '../redirect_to';
 import { useStreamsAppRouter } from '../../hooks/use_streams_app_router';
@@ -25,7 +25,7 @@ export function StreamDetailManagement({
   definition,
   refreshDefinition,
 }: {
-  definition?: StreamDefinition;
+  definition?: APIReturnType<'GET /api/streams/{id}'>;
   refreshDefinition: () => void;
 }) {
   const {

x-pack/plugins/streams_app/public/components/stream_detail_overview/index.tsx

@@ -8,15 +8,19 @@ import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui';
 import { calculateAuto } from '@kbn/calculate-auto';
 import { i18n } from '@kbn/i18n';
 import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range';
-import { StreamDefinition } from '@kbn/streams-plugin/common';
 import moment from 'moment';
 import React, { useMemo } from 'react';
+import type { APIReturnType } from '@kbn/streams-plugin/public/api';
 import { useKibana } from '../../hooks/use_kibana';
 import { useStreamsAppFetch } from '../../hooks/use_streams_app_fetch';
 import { ControlledEsqlChart } from '../esql_chart/controlled_esql_chart';
 import { StreamsAppSearchBar } from '../streams_app_search_bar';
 
-export function StreamDetailOverview({ definition }: { definition?: StreamDefinition }) {
+export function StreamDetailOverview({
+  definition,
+}: {
+  definition?: APIReturnType<'GET /api/streams/{id}'>;
+}) {
   const {
     dependencies: {
       start: {

x-pack/plugins/streams_app/public/components/stream_detail_schema_editor/fields_table.tsx

@@ -0,0 +1,168 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import {
+  EuiBadge,
+  EuiBasicTable,
+  EuiButtonIcon,
+  EuiContextMenu,
+  EuiIcon,
+  EuiPopover,
+  useGeneratedHtmlId,
+} from '@elastic/eui';
+import type {
+  EuiBasicTableColumn,
+  EuiContextMenuPanelDescriptor,
+  EuiContextMenuPanelItemDescriptor,
+} from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import useToggle from 'react-use/lib/useToggle';
+import { FieldDefinition } from '@kbn/streams-plugin/common/types';
+
+export interface FieldEntry {
+  fieldName: FieldDefinition['name'];
+  fieldType?: FieldDefinition['type'];
+  fieldFormat?: string;
+  fieldParent: string;
+}
+
+interface FieldsTableProps {
+  fields: FieldEntry[];
+  actions?: ActionsCellActionsDescriptor[];
+}
+
+export const EMPTY_CONTENT = '-----';
+
+export const FieldsTable = ({ fields, actions }: FieldsTableProps) => {
+  const columns: Array<EuiBasicTableColumn<FieldEntry>> = [
+    {
+      field: 'fieldName',
+      name: 'Field',
+      'data-test-subj': 'fieldName',
+    },
+    {
+      field: 'fieldType',
+      name: 'Type',
+      render: (type: FieldEntry['fieldType']) => {
+        if (!type) return EMPTY_CONTENT;
+        // todo: i18n
+        const capitalisedType = type.charAt(0).toUpperCase() + type.slice(1);
+
+        return type === 'match_only_text' ? (
+          <>
+            <EuiIcon type="tokenText" size="l" />{' '}
+            {i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableTextType', {
+              defaultMessage: 'Text',
+            })}
+          </>
+        ) : ['long', 'double'].includes(type) ? (
+          <>
+            <EuiIcon type="tokenNumber" size="l" />{' '}
+            {i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableNumberType', {
+              defaultMessage: 'Number',
+            })}
+          </>
+        ) : (
+          <>
+            <EuiIcon type={`token${capitalisedType}`} size="l" /> {capitalisedType}
+          </>
+        );
+      },
+    },
+    {
+      field: 'fieldFormat',
+      name: 'Format',
+      render: () => EMPTY_CONTENT, // TODO: Add format support
+    },
+    {
+      field: 'fieldParent',
+      name: 'Field parent',
+      render: (parent: FieldEntry['fieldParent']) => {
+        return <EuiBadge color="hollow">{parent}</EuiBadge>;
+      },
+    },
+    ...(actions
+      ? [
+          {
+            name: 'Actions',
+            render: (field: FieldEntry) => {
+              return (
+                <ActionsCell
+                  panels={[
+                    {
+                      id: 0,
+                      title: i18n.translate(
+                        'xpack.streams.streamDetailSchemaEditorFieldsTableActionsTitle',
+                        {
+                          defaultMessage: 'Actions',
+                        }
+                      ),
+                      items: actions.map((action) => ({
+                        name: action.name,
+                        icon: action.icon,
+                        onClick: (event) => {
+                          action.onClick(field);
+                        },
+                      })),
+                    },
+                  ]}
+                />
+              );
+            },
+          },
+        ]
+      : []),
+  ];
+  return <EuiBasicTable items={fields} rowHeader="name" columns={columns} />;
+};
+
+export const ActionsCell = ({ panels }: { panels: EuiContextMenuPanelDescriptor[] }) => {
+  const contextMenuPopoverId = useGeneratedHtmlId({
+    prefix: 'fieldsTableContextMenuPopover',
+  });
+
+  const [popoverIsOpen, togglePopoverIsOpen] = useToggle(false);
+
+  return (
+    <EuiPopover
+      id={contextMenuPopoverId}
+      button={
+        <EuiButtonIcon
+          data-test-subj="streamsAppActionsButton"
+          iconType="boxesVertical"
+          onClick={() => {
+            togglePopoverIsOpen();
+          }}
+        />
+      }
+      isOpen={popoverIsOpen}
+      closePopover={() => togglePopoverIsOpen(false)}
+    >
+      <EuiContextMenu
+        initialPanelId={0}
+        panels={panels.map((panel) => ({
+          ...panel,
+          items: panel.items?.map((item) => ({
+            name: item.name,
+            icon: item.icon,
+            onClick: (event) => {
+              if (item.onClick) {
+                item.onClick(event as any);
+              }
+              togglePopoverIsOpen(false);
+            },
+          })),
+        }))}
+      />
+    </EuiPopover>
+  );
+};
+
+export type ActionsCellActionsDescriptor = Omit<EuiContextMenuPanelItemDescriptor, 'onClick'> & {
+  onClick: (field: FieldEntry) => void;
+};

x-pack/plugins/streams_app/public/components/stream_detail_schema_editor/flyout/field_form.tsx

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiForm } from '@elastic/eui';
+import React from 'react';
+import { SchemaEditorEditingState } from '../hooks/use_editing_state';
+import { FieldFormType } from './field_form_type';
+import { FieldFormFormat } from './field_form_format';
+
+type FieldFormProps = SchemaEditorEditingState;
+
+export const FieldForm = ({
+  nextFieldType,
+  setNextFieldType,
+  nextFieldFormat,
+  setNextFieldFormat,
+}: FieldFormProps) => {
+  return (
+    <EuiForm component="form">
+      <FieldFormType nextFieldType={nextFieldType} setNextFieldType={setNextFieldType} />
+      <FieldFormFormat
+        nextFieldType={nextFieldType}
+        nextFieldFormat={nextFieldFormat}
+        setNextFieldFormat={setNextFieldFormat}
+      />
+    </EuiForm>
+  );
+};

x-pack/plugins/streams_app/public/components/stream_detail_schema_editor/flyout/field_form_format.tsx

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiFieldText, EuiFormRow } from '@elastic/eui';
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { FieldDefinition } from '@kbn/streams-plugin/common/types';
+import { SchemaEditorEditingState } from '../hooks/use_editing_state';
+
+type FieldFormFormatProps = Pick<
+  SchemaEditorEditingState,
+  'nextFieldType' | 'nextFieldFormat' | 'setNextFieldFormat'
+>;
+
+const typeSupportsFormat = (type?: FieldDefinition['type']) => {
+  if (!type) return false;
+  return ['date'].includes(type);
+};
+
+export const FieldFormFormat = ({
+  nextFieldType: fieldType,
+  nextFieldFormat: value,
+  setNextFieldFormat: onChange,
+}: FieldFormFormatProps) => {
+  if (!typeSupportsFormat(fieldType)) {
+    return null;
+  }
+  return (
+    <EuiFormRow
+      label={i18n.translate('xpack.streams.fieldForm.format.label', {
+        defaultMessage: 'Format',
+      })}
+    >
+      <EuiFieldText
+        data-test-subj="streamsAppFieldFormFormatField"
+        placeholder="yyyy/MM/dd"
+        value={value}
+        onChange={(e) => onChange(e.target.value)}
+      />
+    </EuiFormRow>
+  );
+};