Skip to content

Commit

Permalink
[Automatic Import] Fix missing ecs mappings (#209057)
Browse files Browse the repository at this point in the history
## Summary

Fix missing ecs mappings.

## Details

If the input is of a multilevel json where there were no ecs mappings
identified on first level, For ex:

```
{
  okta_test: {
    audit: {
      actor: [Object],
      authenticationContext: [Object],
      client: [Object],
      debugContext: [Object],
      displayMessage: [Object],
      eventType: [Object],
      legacyEventType: null,
      outcome: [Object],
      published: [Object],
      request: [Object],
      securityContext: [Object],
      severity: [Object],
      target: null,
      transaction: [Object],
      uuid: [Object],
      version: null
    }
  }
}
```

Then there is no `target` field identified and an iteration is triggered
to find the ecs mappings in the next level. Since there is no `target`
identified the logic exited and further levels were skipped in
identifying the mappings.

This PR fixes it and skips handling of Arrays since `rename` processor
cannot flatten arrays on its own to map searchable fields. This will
handled separately.
  • Loading branch information
bhapas authored Jan 31, 2025
1 parent a724cdf commit 2fe3976
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import { ecsPipelineState } from '../../../__jest__/fixtures/ecs_mapping';
import type { EcsMappingState } from '../../types';
import { createPipeline } from './pipeline';
import { createPipeline, generateProcessors } from './pipeline';

const state: EcsMappingState = ecsPipelineState;

Expand Down Expand Up @@ -303,4 +303,118 @@ describe('Testing pipeline templates', () => {
},
]);
});

it('should generate processors for empty mapping', () => {
const processors = generateProcessors({}, {});
expect(processors).toEqual([]);
});

it('should generate processors for nested fields', () => {
const mapping = {
nested: {
field: {
target: 'target.field',
confidence: 0.8,
type: 'keyword',
date_formats: [],
},
},
};
const samples = {
nested: {
field: 'test value',
},
};

const processors = generateProcessors(mapping, samples);

expect(processors).toHaveLength(1);
expect(processors[0]).toEqual({
rename: {
field: 'nested.field',
target_field: 'target.field',
ignore_missing: true,
},
});
});
it('should handle nested fields with mixed target mappings', () => {
const mapping = {
level1: {
no_target: {
field: {
target: 'target.field',
confidence: 0.8,
type: 'keyword',
date_formats: [],
},
},
},
};
const samples = {
level1: {
no_target: {
field: 'test value',
},
},
};

const processors = generateProcessors(mapping, samples);

expect(processors).toHaveLength(1);
expect(processors[0]).toEqual({
rename: {
field: 'level1.no_target.field',
target_field: 'target.field',
ignore_missing: true,
},
});
});
it('should handle multiple nested fields', () => {
const mapping = {
level1: {
level2: {
level3: {
field1: {
target: 'target.field1',
confidence: 0.8,
type: 'keyword',
date_formats: [],
},
},
field2: {
target: 'target.field2',
confidence: 0.9,
type: 'long',
date_formats: [],
},
},
},
};
const samples = {
level1: {
level2: {
field1: 'string value',
field2: '123',
},
},
};

const processors = generateProcessors(mapping, samples);

expect(processors).toHaveLength(2);
expect(processors[0]).toEqual({
rename: {
field: 'level1.level2.level3.field1',
target_field: 'target.field1',
ignore_missing: true,
},
});
expect(processors[1]).toEqual({
rename: {
field: 'level1.level2.field2',
target_field: 'target.field2',
ignore_missing: true,
},
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ function needsTypeConversion(sample: unknown, expected: KnownESType): boolean {
return false;
}

function generateProcessors(
export function generateProcessors(
ecsMapping: object,
samples: object,
basePath: FieldPath = []
Expand All @@ -167,10 +167,9 @@ function generateProcessors(

for (const [key, value] of Object.entries(ecsMapping)) {
const currentPath = [...basePath, key];

if (value !== null && typeof value === 'object' && value?.target !== null) {
if (value !== null && !Array.isArray(value) && typeof value === 'object') {
const valueKeys = new Set(Object.keys(value));
if ([...valueFieldKeys].every((k) => valueKeys.has(k))) {
if (value?.target != null && [...valueFieldKeys].every((k) => valueKeys.has(k))) {
const processor = generateProcessor(
currentPath,
value as ECSField,
Expand Down

0 comments on commit 2fe3976

Please sign in to comment.