From 2fe39767eeee79e7e059afe5a07d7724e80c0969 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com>
Date: Fri, 31 Jan 2025 12:04:45 +0100
Subject: [PATCH] [Automatic Import] Fix missing ecs mappings (#209057)

## Summary

Fix missing ecs mappings.

## Details

If the input is of a multilevel json where there were no ecs mappings
identified on first level, For ex:

```
{
  okta_test: {
    audit: {
      actor: [Object],
      authenticationContext: [Object],
      client: [Object],
      debugContext: [Object],
      displayMessage: [Object],
      eventType: [Object],
      legacyEventType: null,
      outcome: [Object],
      published: [Object],
      request: [Object],
      securityContext: [Object],
      severity: [Object],
      target: null,
      transaction: [Object],
      uuid: [Object],
      version: null
    }
  }
}
```

Then there is no `target` field identified and an iteration is triggered
to find the ecs mappings in the next level. Since there is no `target`
identified the logic exited and further levels were skipped in
identifying the mappings.

This PR fixes it and skips handling of Arrays since `rename` processor
cannot flatten arrays on its own to map searchable fields. This will
handled separately.
---
 .../server/graphs/ecs/pipeline.test.ts        | 116 +++++++++++++++++-
 .../server/graphs/ecs/pipeline.ts             |   7 +-
 2 files changed, 118 insertions(+), 5 deletions(-)

diff --git a/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.test.ts b/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.test.ts
index 0ad142390b865..01c5a9c056205 100644
--- a/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.test.ts
+++ b/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.test.ts
@@ -7,7 +7,7 @@
 
 import { ecsPipelineState } from '../../../__jest__/fixtures/ecs_mapping';
 import type { EcsMappingState } from '../../types';
-import { createPipeline } from './pipeline';
+import { createPipeline, generateProcessors } from './pipeline';
 
 const state: EcsMappingState = ecsPipelineState;
 
@@ -303,4 +303,118 @@ describe('Testing pipeline templates', () => {
       },
     ]);
   });
+
+  it('should generate processors for empty mapping', () => {
+    const processors = generateProcessors({}, {});
+    expect(processors).toEqual([]);
+  });
+
+  it('should generate processors for nested fields', () => {
+    const mapping = {
+      nested: {
+        field: {
+          target: 'target.field',
+          confidence: 0.8,
+          type: 'keyword',
+          date_formats: [],
+        },
+      },
+    };
+    const samples = {
+      nested: {
+        field: 'test value',
+      },
+    };
+
+    const processors = generateProcessors(mapping, samples);
+
+    expect(processors).toHaveLength(1);
+    expect(processors[0]).toEqual({
+      rename: {
+        field: 'nested.field',
+        target_field: 'target.field',
+        ignore_missing: true,
+      },
+    });
+  });
+  it('should handle nested fields with mixed target mappings', () => {
+    const mapping = {
+      level1: {
+        no_target: {
+          field: {
+            target: 'target.field',
+            confidence: 0.8,
+            type: 'keyword',
+            date_formats: [],
+          },
+        },
+      },
+    };
+    const samples = {
+      level1: {
+        no_target: {
+          field: 'test value',
+        },
+      },
+    };
+
+    const processors = generateProcessors(mapping, samples);
+
+    expect(processors).toHaveLength(1);
+    expect(processors[0]).toEqual({
+      rename: {
+        field: 'level1.no_target.field',
+        target_field: 'target.field',
+        ignore_missing: true,
+      },
+    });
+  });
+  it('should handle multiple nested fields', () => {
+    const mapping = {
+      level1: {
+        level2: {
+          level3: {
+            field1: {
+              target: 'target.field1',
+              confidence: 0.8,
+              type: 'keyword',
+              date_formats: [],
+            },
+          },
+          field2: {
+            target: 'target.field2',
+            confidence: 0.9,
+            type: 'long',
+            date_formats: [],
+          },
+        },
+      },
+    };
+    const samples = {
+      level1: {
+        level2: {
+          field1: 'string value',
+          field2: '123',
+        },
+      },
+    };
+
+    const processors = generateProcessors(mapping, samples);
+
+    expect(processors).toHaveLength(2);
+    expect(processors[0]).toEqual({
+      rename: {
+        field: 'level1.level2.level3.field1',
+        target_field: 'target.field1',
+        ignore_missing: true,
+      },
+    });
+    expect(processors[1]).toEqual({
+      rename: {
+        field: 'level1.level2.field2',
+        target_field: 'target.field2',
+        ignore_missing: true,
+      },
+    });
+  });
 });
diff --git a/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.ts b/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.ts
index d729df1edde3d..9d5342eb1b370 100644
--- a/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.ts
+++ b/x-pack/platform/plugins/shared/automatic_import/server/graphs/ecs/pipeline.ts
@@ -153,7 +153,7 @@ function needsTypeConversion(sample: unknown, expected: KnownESType): boolean {
   return false;
 }
 
-function generateProcessors(
+export function generateProcessors(
   ecsMapping: object,
   samples: object,
   basePath: FieldPath = []
@@ -167,10 +167,9 @@ function generateProcessors(
 
   for (const [key, value] of Object.entries(ecsMapping)) {
     const currentPath = [...basePath, key];
-
-    if (value !== null && typeof value === 'object' && value?.target !== null) {
+    if (value !== null && !Array.isArray(value) && typeof value === 'object') {
       const valueKeys = new Set(Object.keys(value));
-      if ([...valueFieldKeys].every((k) => valueKeys.has(k))) {
+      if (value?.target != null && [...valueFieldKeys].every((k) => valueKeys.has(k))) {
         const processor = generateProcessor(
           currentPath,
           value as ECSField,