diff --git a/logprep/processor/generic_resolver/processor.py b/logprep/processor/generic_resolver/processor.py index fec01512c..9315713fe 100644 --- a/logprep/processor/generic_resolver/processor.py +++ b/logprep/processor/generic_resolver/processor.py @@ -27,9 +27,10 @@ import re +from logprep.processor.base.exceptions import FieldExistsWarning from logprep.processor.field_manager.processor import FieldManager from logprep.processor.generic_resolver.rule import GenericResolverRule -from logprep.util.helper import add_field_to, get_dotted_field_value +from logprep.util.helper import get_dotted_field_value, add_field_to_silent_fail class GenericResolver(FieldManager): @@ -44,6 +45,7 @@ def _apply_rules(self, event, rule): for source_field in rule.field_mapping.keys() ] self._handle_missing_fields(event, rule, rule.field_mapping.keys(), source_field_values) + conflicting_fields = [] for source_field, target_field in rule.field_mapping.items(): source_field_value = get_dotted_field_value(event, source_field) if source_field_value is None: @@ -56,13 +58,17 @@ def _apply_rules(self, event, rule): continue if rule.extend_target_list and current_content is None: content = [content] - add_field_to( + failed_target = add_field_to_silent_fail( event, target_field, content, extends_lists=rule.extend_target_list, overwrite_output_field=rule.overwrite_target, ) + if failed_target: + conflicting_fields.append(failed_target) + if conflicting_fields: + raise FieldExistsWarning(event, conflicting_fields, rule) def _find_content_of_first_matching_pattern(self, rule, source_field_value): if rule.resolve_from_file: diff --git a/logprep/processor/hyperscan_resolver/processor.py b/logprep/processor/hyperscan_resolver/processor.py index d942d1ee2..8dedbc4a1 100644 --- a/logprep/processor/hyperscan_resolver/processor.py +++ b/logprep/processor/hyperscan_resolver/processor.py @@ -33,7 +33,7 @@ import errno from os import makedirs, path -from typing import Any, Dict, Tuple, Union +from typing import Any, Dict, Tuple from attr import define, field @@ -43,7 +43,7 @@ ProcessingCriticalError, ) from logprep.processor.field_manager.processor import FieldManager -from logprep.util.helper import add_field_to, get_dotted_field_value +from logprep.util.helper import get_dotted_field_value, add_field_to_silent_fail from logprep.util.validators import directory_validator # pylint: disable=no-name-in-module @@ -57,6 +57,7 @@ # pylint: disable=ungrouped-imports from logprep.processor.hyperscan_resolver.rule import HyperscanResolverRule + # pylint: enable=ungrouped-imports @@ -113,39 +114,23 @@ def _apply_rules(self, event: dict, rule: HyperscanResolverRule): if matches: dest_val = pattern_id_to_dest_val_map[matches[matches.index(min(matches))]] if dest_val: - add_success = self._add_uniquely_to_list(event, rule, resolve_target, dest_val) - if not add_success: - conflicting_fields.append(resolve_target) + current_content = get_dotted_field_value(event, resolve_target) + if isinstance(current_content, list) and dest_val in current_content: + continue + if rule.extend_target_list and current_content is None: + dest_val = [dest_val] + failed_target = add_field_to_silent_fail( + event, + resolve_target, + dest_val, + extends_lists=rule.extend_target_list, + overwrite_output_field=rule.overwrite_target, + ) + if failed_target: + conflicting_fields.append(failed_target) self._handle_missing_fields(event, rule, rule.field_mapping.keys(), source_values) if conflicting_fields: - raise FieldExistsWarning(rule, event, conflicting_fields) - - @staticmethod - def _add_uniquely_to_list( - event: dict, - rule: HyperscanResolverRule, - target: str, - content: Union[str, float, int, list, dict], - ) -> bool: - """Extend list if content is not already in the list""" - add_success = True - target_val = get_dotted_field_value(event, target) - target_is_list = isinstance(target_val, list) - if rule.extend_target_list and not target_is_list: - empty_list = [] - add_success &= add_field_to( - event, - target, - empty_list, - overwrite_output_field=rule.overwrite_target, - ) - if add_success: - target_is_list = True - target_val = empty_list - if target_is_list and content in target_val: - return add_success - add_success = add_field_to(event, target, content, extends_lists=rule.extend_target_list) - return add_success + raise FieldExistsWarning(event, conflicting_fields, rule) @staticmethod def _match_with_hyperscan(hyperscan_db: Database, src_val: str) -> list: diff --git a/logprep/util/helper.py b/logprep/util/helper.py index e18be080a..16c5efc31 100644 --- a/logprep/util/helper.py +++ b/logprep/util/helper.py @@ -59,9 +59,9 @@ def _add_and_not_overwrite_key(sub_dict, key): return sub_dict.get(key) -def add_field_to_silent_fail(*args): +def add_field_to_silent_fail(*args, **kwargs): try: - add_field_to(*args) + add_field_to(*args, **kwargs) except FieldExistsWarning: return args[1]