From 5c1373d8a145fd02a1570a53e9f604c72ba57621 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 30 Jul 2024 13:41:09 +0930 Subject: [PATCH] symantec_endpoint_security: handle non-map raw data values (#10630) --- .../symantec_endpoint_security/changelog.yml | 5 + .../test/pipeline/test-scalar-flattened.log | 2 + .../test-scalar-flattened.log-expected.json | 264 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 19 ++ .../symantec_endpoint_security/manifest.yml | 2 +- 5 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log create mode 100644 packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log-expected.json diff --git a/packages/symantec_endpoint_security/changelog.yml b/packages/symantec_endpoint_security/changelog.yml index ad14e811499..94914198100 100644 --- a/packages/symantec_endpoint_security/changelog.yml +++ b/packages/symantec_endpoint_security/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.1" + changes: + - description: Improve handling of scalar `raw_data` field values. + type: bugfix + link: https://github.com/elastic/integrations/pull/10630 - version: "0.3.0" changes: - description: Merge Symantec EDR Cloud into Symantec Endpoint Security. diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log b/packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log new file mode 100644 index 00000000000..3363288e7a5 --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log @@ -0,0 +1,2 @@ +{"category_id":1,"count":1,"device_end_time":1721812942435,"device_ip":"192.168.100.100","device_location":{"desc":"Default","on_premises":false },"device_time":1721812956826,"feature_name":"COMPLIANCE","feature_uid":"123E4567-E89B-12D3-A456-426614174000","id":1,"message":"Host Integrity check passed\n Requirement:\"Symantec Definition up-to-date?\" passed\n","policy":{"name":"Default Host Integrity","uid":"123e4567-e89b-12d3-a456-426614174001","version":"3"},"raw_data":"46563D312E300A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D73746172740A433D61765F7369676E61747572655F6F6B5E533D706173735E543D2253796D616E74656320456E64706F696E742050726F74656374696F6E220A433D74696D657374616D705F6F6B5E543D22436865636B2054696D657374616D70225E533D706173730A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D70617373","scan_end":1721812942435,"scan_start":1721812942435,"scan_type_id":2,"scan_uid":"123E4567E89B12D3A456426614174002","severity_id":1,"subfeature_name":"Host Integrity","type":"HOST_COMPLIANCE_SCAN","type_id":8070,"version":"1.0","composite":2,"device_domain":"example.com","device_group":"Default/Example/XXX-US-Example","device_name":"DESKTOP-EXAMPLE","device_networks":[{"ipv4":"192.168.100.101","ipv6":"fe80::26a3:da5d:4c46:523d","mac":"00:05:9A:3C:7A:00"}],"device_os_name":"Windows 11 Professional Edition","device_uid":"123e4567E89b12d3a45642","org_unit_uid":"123e4567s_E89be89be89Q","product_data":{"sep_domain_uid":"","sep_hw_uid":"123E4567E89B12D3A456426614174005"},"product_name":"Symantec Endpoint Security","product_uid":"123E4567-E89B-12D3-A456-426614174006","product_ver":"14.3.11216.9000","stic_hw_uid":"123E4567-E89B-12D3-A456-426614174007","stic_uid":"123E4567-E89B-12D3-A456-426614174008","timezone":-120,"user_name":"user123456","customer_uid":"123e4567e89b12d3a45642","device_public_ip":"81.2.69.144","domain_uid":"123e4567e89b12d3a45642","user":{"name":"user123456"},"device_os_type_id":100,"time":"2024-07-24T09:22:36.826Z","end_time":"2024-07-24T09:22:22.435Z","log_time":"2024-07-24T09:22:43.530Z","uuid":"8070:123e4567-e89b-12d3-a456-426614174011"} +{"category_id":1,"count":1,"device_end_time":1721812942435,"device_ip":"192.168.100.100","device_location":{"desc":"Default","on_premises":false },"device_time":1721812956826,"feature_name":"COMPLIANCE","feature_uid":"123E4567-E89B-12D3-A456-426614174000","id":1,"message":"Host Integrity check passed\n Requirement:\"Symantec Definition up-to-date?\" passed\n","policy":{"name":"Default Host Integrity","uid":"123e4567-e89b-12d3-a456-426614174001","version":"3"},"raw_data":["46563D312E300A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D73746172740A433D61765F7369676E61747572655F6F6B5E533D706173735E543D2253796D616E74656320456E64706F696E742050726F74656374696F6E220A433D74696D657374616D705F6F6B5E543D22436865636B2054696D657374616D70225E533D706173730A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D70617373"],"scan_end":1721812942435,"scan_start":1721812942435,"scan_type_id":2,"scan_uid":"123E4567E89B12D3A456426614174002","severity_id":1,"subfeature_name":"Host Integrity","type":"HOST_COMPLIANCE_SCAN","type_id":8070,"version":"1.0","composite":2,"device_domain":"example.com","device_group":"Default/Example/XXX-US-Example","device_name":"DESKTOP-EXAMPLE","device_networks":[{"ipv4":"192.168.100.101","ipv6":"fe80::26a3:da5d:4c46:523d","mac":"00:05:9A:3C:7A:00"}],"device_os_name":"Windows 11 Professional Edition","device_uid":"123e4567E89b12d3a45642","org_unit_uid":"123e4567s_E89be89be89Q","product_data":{"sep_domain_uid":"","sep_hw_uid":"123E4567E89B12D3A456426614174005"},"product_name":"Symantec Endpoint Security","product_uid":"123E4567-E89B-12D3-A456-426614174006","product_ver":"14.3.11216.9000","stic_hw_uid":"123E4567-E89B-12D3-A456-426614174007","stic_uid":"123E4567-E89B-12D3-A456-426614174008","timezone":-120,"user_name":"user123456","customer_uid":"123e4567e89b12d3a45642","device_public_ip":"81.2.69.144","domain_uid":"123e4567e89b12d3a45642","user":{"name":"user123456"},"device_os_type_id":100,"time":"2024-07-24T09:22:36.826Z","end_time":"2024-07-24T09:22:22.435Z","log_time":"2024-07-24T09:22:43.530Z","uuid":"8070:123e4567-e89b-12d3-a456-426614174011"} diff --git a/packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log-expected.json b/packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log-expected.json new file mode 100644 index 00000000000..d0cb3683473 --- /dev/null +++ b/packages/symantec_endpoint_security/data_stream/event/_dev/test/pipeline/test-scalar-flattened.log-expected.json @@ -0,0 +1,264 @@ +{ + "expected": [ + { + "@timestamp": "2024-07-24T09:22:36.826Z", + "client": { + "domain": "example.com" + }, + "device": { + "id": [ + "123e4567E89b12d3a45642" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "created": "2024-07-24T09:22:36.826Z", + "end": [ + "2024-07-24T09:22:22.435Z" + ], + "id": "8070:123e4567-e89b-12d3-a456-426614174011", + "kind": "event", + "original": "{\"category_id\":1,\"count\":1,\"device_end_time\":1721812942435,\"device_ip\":\"192.168.100.100\",\"device_location\":{\"desc\":\"Default\",\"on_premises\":false },\"device_time\":1721812956826,\"feature_name\":\"COMPLIANCE\",\"feature_uid\":\"123E4567-E89B-12D3-A456-426614174000\",\"id\":1,\"message\":\"Host Integrity check passed\\n Requirement:\\\"Symantec Definition up-to-date?\\\" passed\\n\",\"policy\":{\"name\":\"Default Host Integrity\",\"uid\":\"123e4567-e89b-12d3-a456-426614174001\",\"version\":\"3\"},\"raw_data\":\"46563D312E300A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D73746172740A433D61765F7369676E61747572655F6F6B5E533D706173735E543D2253796D616E74656320456E64706F696E742050726F74656374696F6E220A433D74696D657374616D705F6F6B5E543D22436865636B2054696D657374616D70225E533D706173730A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D70617373\",\"scan_end\":1721812942435,\"scan_start\":1721812942435,\"scan_type_id\":2,\"scan_uid\":\"123E4567E89B12D3A456426614174002\",\"severity_id\":1,\"subfeature_name\":\"Host Integrity\",\"type\":\"HOST_COMPLIANCE_SCAN\",\"type_id\":8070,\"version\":\"1.0\",\"composite\":2,\"device_domain\":\"example.com\",\"device_group\":\"Default/Example/XXX-US-Example\",\"device_name\":\"DESKTOP-EXAMPLE\",\"device_networks\":[{\"ipv4\":\"192.168.100.101\",\"ipv6\":\"fe80::26a3:da5d:4c46:523d\",\"mac\":\"00:05:9A:3C:7A:00\"}],\"device_os_name\":\"Windows 11 Professional Edition\",\"device_uid\":\"123e4567E89b12d3a45642\",\"org_unit_uid\":\"123e4567s_E89be89be89Q\",\"product_data\":{\"sep_domain_uid\":\"\",\"sep_hw_uid\":\"123E4567E89B12D3A456426614174005\"},\"product_name\":\"Symantec Endpoint Security\",\"product_uid\":\"123E4567-E89B-12D3-A456-426614174006\",\"product_ver\":\"14.3.11216.9000\",\"stic_hw_uid\":\"123E4567-E89B-12D3-A456-426614174007\",\"stic_uid\":\"123E4567-E89B-12D3-A456-426614174008\",\"timezone\":-120,\"user_name\":\"user123456\",\"customer_uid\":\"123e4567e89b12d3a45642\",\"device_public_ip\":\"81.2.69.144\",\"domain_uid\":\"123e4567e89b12d3a45642\",\"user\":{\"name\":\"user123456\"},\"device_os_type_id\":100,\"time\":\"2024-07-24T09:22:36.826Z\",\"end_time\":\"2024-07-24T09:22:22.435Z\",\"log_time\":\"2024-07-24T09:22:43.530Z\",\"uuid\":\"8070:123e4567-e89b-12d3-a456-426614174011\"}", + "severity": 1, + "start": [ + "2024-07-24T09:22:22.435Z" + ] + }, + "host": { + "os": { + "name": "Windows 11 Professional Edition", + "type": [ + "windows" + ] + } + }, + "message": "Host Integrity check passed\n Requirement:\"Symantec Definition up-to-date?\" passed\n", + "related": { + "ip": [ + "192.168.100.100", + "81.2.69.144", + "192.168.100.101", + "fe80::26a3:da5d:4c46:523d" + ], + "user": [ + "user123456" + ] + }, + "ses": { + "category_id": "1", + "category_name": "Security", + "composite": 2, + "count": 1, + "customer_uid": "123e4567e89b12d3a45642", + "device_domain": "example.com", + "device_end_time": "2024-07-24T09:22:22.435Z", + "device_group": "Default/Example/XXX-US-Example", + "device_ip": "192.168.100.100", + "device_location": { + "desc": "Default", + "on_premises": false + }, + "device_name": "DESKTOP-EXAMPLE", + "device_networks": [ + { + "ipv4": "192.168.100.101", + "ipv6": "fe80::26a3:da5d:4c46:523d", + "mac": "00:05:9A:3C:7A:00" + } + ], + "device_os_name": "Windows 11 Professional Edition", + "device_os_type_id": "100", + "device_os_type_value": "Windows", + "device_public_ip": "81.2.69.144", + "device_time": "2024-07-24T09:22:36.826Z", + "device_uid": "123e4567E89b12d3a45642", + "domain_uid": "123e4567e89b12d3a45642", + "end_time": "2024-07-24T09:22:22.435Z", + "feature_name": "COMPLIANCE", + "feature_uid": "123E4567-E89B-12D3-A456-426614174000", + "id": 1, + "log_time": "2024-07-24T09:22:43.530Z", + "message": "Host Integrity check passed\n Requirement:\"Symantec Definition up-to-date?\" passed\n", + "org_unit_uid": "123e4567s_E89be89be89Q", + "policy": { + "name": "Default Host Integrity", + "uid": "123e4567-e89b-12d3-a456-426614174001", + "version": "3" + }, + "product_data": { + "sep_hw_uid": "123E4567E89B12D3A456426614174005" + }, + "product_name": "Symantec Endpoint Security", + "product_uid": "123E4567-E89B-12D3-A456-426614174006", + "product_ver": "14.3.11216.9000", + "raw_data": { + "value": "46563D312E300A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D73746172740A433D61765F7369676E61747572655F6F6B5E533D706173735E543D2253796D616E74656320456E64706F696E742050726F74656374696F6E220A433D74696D657374616D705F6F6B5E543D22436865636B2054696D657374616D70225E533D706173730A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D70617373" + }, + "scan_end": "2024-07-24T09:22:22.435Z", + "scan_start": "2024-07-24T09:22:22.435Z", + "scan_type_id": "2", + "scan_type_value": "Scheduled", + "scan_uid": "123E4567E89B12D3A456426614174002", + "severity_id": "1", + "severity_value": "Informational", + "stic_hw_uid": "123E4567-E89B-12D3-A456-426614174007", + "stic_uid": "123E4567-E89B-12D3-A456-426614174008", + "subfeature_name": "Host Integrity", + "time": "2024-07-24T09:22:36.826Z", + "timezone": -120, + "type": "HOST_COMPLIANCE_SCAN", + "type_id": "8070", + "user": { + "name": "user123456" + }, + "user_name": "user123456", + "uuid": "8070:123e4567-e89b-12d3-a456-426614174011", + "version": "1.0" + }, + "source": { + "address": "DESKTOP-EXAMPLE", + "ip": "192.168.100.100" + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ], + "user": { + "name": "user123456" + } + }, + { + "@timestamp": "2024-07-24T09:22:36.826Z", + "client": { + "domain": "example.com" + }, + "device": { + "id": [ + "123e4567E89b12d3a45642" + ] + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "created": "2024-07-24T09:22:36.826Z", + "end": [ + "2024-07-24T09:22:22.435Z" + ], + "id": "8070:123e4567-e89b-12d3-a456-426614174011", + "kind": "event", + "original": "{\"category_id\":1,\"count\":1,\"device_end_time\":1721812942435,\"device_ip\":\"192.168.100.100\",\"device_location\":{\"desc\":\"Default\",\"on_premises\":false },\"device_time\":1721812956826,\"feature_name\":\"COMPLIANCE\",\"feature_uid\":\"123E4567-E89B-12D3-A456-426614174000\",\"id\":1,\"message\":\"Host Integrity check passed\\n Requirement:\\\"Symantec Definition up-to-date?\\\" passed\\n\",\"policy\":{\"name\":\"Default Host Integrity\",\"uid\":\"123e4567-e89b-12d3-a456-426614174001\",\"version\":\"3\"},\"raw_data\":[\"46563D312E300A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D73746172740A433D61765F7369676E61747572655F6F6B5E533D706173735E543D2253796D616E74656320456E64706F696E742050726F74656374696F6E220A433D74696D657374616D705F6F6B5E543D22436865636B2054696D657374616D70225E533D706173730A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D70617373\"],\"scan_end\":1721812942435,\"scan_start\":1721812942435,\"scan_type_id\":2,\"scan_uid\":\"123E4567E89B12D3A456426614174002\",\"severity_id\":1,\"subfeature_name\":\"Host Integrity\",\"type\":\"HOST_COMPLIANCE_SCAN\",\"type_id\":8070,\"version\":\"1.0\",\"composite\":2,\"device_domain\":\"example.com\",\"device_group\":\"Default/Example/XXX-US-Example\",\"device_name\":\"DESKTOP-EXAMPLE\",\"device_networks\":[{\"ipv4\":\"192.168.100.101\",\"ipv6\":\"fe80::26a3:da5d:4c46:523d\",\"mac\":\"00:05:9A:3C:7A:00\"}],\"device_os_name\":\"Windows 11 Professional Edition\",\"device_uid\":\"123e4567E89b12d3a45642\",\"org_unit_uid\":\"123e4567s_E89be89be89Q\",\"product_data\":{\"sep_domain_uid\":\"\",\"sep_hw_uid\":\"123E4567E89B12D3A456426614174005\"},\"product_name\":\"Symantec Endpoint Security\",\"product_uid\":\"123E4567-E89B-12D3-A456-426614174006\",\"product_ver\":\"14.3.11216.9000\",\"stic_hw_uid\":\"123E4567-E89B-12D3-A456-426614174007\",\"stic_uid\":\"123E4567-E89B-12D3-A456-426614174008\",\"timezone\":-120,\"user_name\":\"user123456\",\"customer_uid\":\"123e4567e89b12d3a45642\",\"device_public_ip\":\"81.2.69.144\",\"domain_uid\":\"123e4567e89b12d3a45642\",\"user\":{\"name\":\"user123456\"},\"device_os_type_id\":100,\"time\":\"2024-07-24T09:22:36.826Z\",\"end_time\":\"2024-07-24T09:22:22.435Z\",\"log_time\":\"2024-07-24T09:22:43.530Z\",\"uuid\":\"8070:123e4567-e89b-12d3-a456-426614174011\"}", + "severity": 1, + "start": [ + "2024-07-24T09:22:22.435Z" + ] + }, + "host": { + "os": { + "name": "Windows 11 Professional Edition", + "type": [ + "windows" + ] + } + }, + "message": "Host Integrity check passed\n Requirement:\"Symantec Definition up-to-date?\" passed\n", + "related": { + "ip": [ + "192.168.100.100", + "81.2.69.144", + "192.168.100.101", + "fe80::26a3:da5d:4c46:523d" + ], + "user": [ + "user123456" + ] + }, + "ses": { + "category_id": "1", + "category_name": "Security", + "composite": 2, + "count": 1, + "customer_uid": "123e4567e89b12d3a45642", + "device_domain": "example.com", + "device_end_time": "2024-07-24T09:22:22.435Z", + "device_group": "Default/Example/XXX-US-Example", + "device_ip": "192.168.100.100", + "device_location": { + "desc": "Default", + "on_premises": false + }, + "device_name": "DESKTOP-EXAMPLE", + "device_networks": [ + { + "ipv4": "192.168.100.101", + "ipv6": "fe80::26a3:da5d:4c46:523d", + "mac": "00:05:9A:3C:7A:00" + } + ], + "device_os_name": "Windows 11 Professional Edition", + "device_os_type_id": "100", + "device_os_type_value": "Windows", + "device_public_ip": "81.2.69.144", + "device_time": "2024-07-24T09:22:36.826Z", + "device_uid": "123e4567E89b12d3a45642", + "domain_uid": "123e4567e89b12d3a45642", + "end_time": "2024-07-24T09:22:22.435Z", + "feature_name": "COMPLIANCE", + "feature_uid": "123E4567-E89B-12D3-A456-426614174000", + "id": 1, + "log_time": "2024-07-24T09:22:43.530Z", + "message": "Host Integrity check passed\n Requirement:\"Symantec Definition up-to-date?\" passed\n", + "org_unit_uid": "123e4567s_E89be89be89Q", + "policy": { + "name": "Default Host Integrity", + "uid": "123e4567-e89b-12d3-a456-426614174001", + "version": "3" + }, + "product_data": { + "sep_hw_uid": "123E4567E89B12D3A456426614174005" + }, + "product_name": "Symantec Endpoint Security", + "product_uid": "123E4567-E89B-12D3-A456-426614174006", + "product_ver": "14.3.11216.9000", + "raw_data": { + "value": [ + "46563D312E300A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D73746172740A433D61765F7369676E61747572655F6F6B5E533D706173735E543D2253796D616E74656320456E64706F696E742050726F74656374696F6E220A433D74696D657374616D705F6F6B5E543D22436865636B2054696D657374616D70225E533D706173730A523D2253796D616E74656320446566696E6974696F6E2075702D746F2D646174653F225E533D70617373" + ] + }, + "scan_end": "2024-07-24T09:22:22.435Z", + "scan_start": "2024-07-24T09:22:22.435Z", + "scan_type_id": "2", + "scan_type_value": "Scheduled", + "scan_uid": "123E4567E89B12D3A456426614174002", + "severity_id": "1", + "severity_value": "Informational", + "stic_hw_uid": "123E4567-E89B-12D3-A456-426614174007", + "stic_uid": "123E4567-E89B-12D3-A456-426614174008", + "subfeature_name": "Host Integrity", + "time": "2024-07-24T09:22:36.826Z", + "timezone": -120, + "type": "HOST_COMPLIANCE_SCAN", + "type_id": "8070", + "user": { + "name": "user123456" + }, + "user_name": "user123456", + "uuid": "8070:123e4567-e89b-12d3-a456-426614174011", + "version": "1.0" + }, + "source": { + "address": "DESKTOP-EXAMPLE", + "ip": "192.168.100.100" + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ], + "user": { + "name": "user123456" + } + } + ] +} \ No newline at end of file diff --git a/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 379fad8b80e..1c3dba50227 100644 --- a/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -1105,6 +1105,25 @@ processors: - append: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - script: + description: Move raw_data to raw_data.value if the value is not compatible with flattened. + lang: painless + if: ctx.ses?.raw_data != null && !(ctx.ses.raw_data instanceof Map) + source: |- + if (ctx.ses.raw_data instanceof List) { + for (def e: ctx.ses.raw_data) { + if (!(e instanceof Map)) { + def m = new HashMap(); + m["value"] = ctx.ses.raw_data; + ctx.ses.raw_data = m; + break; + } + } + return; + } + def m = new HashMap(); + m["value"] = ctx.ses.raw_data; + ctx.ses.raw_data = m; - remove: field: - ses.collector_device_ip diff --git a/packages/symantec_endpoint_security/manifest.yml b/packages/symantec_endpoint_security/manifest.yml index 801e3b4a072..58d0d8b3664 100644 --- a/packages/symantec_endpoint_security/manifest.yml +++ b/packages/symantec_endpoint_security/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.3 name: symantec_endpoint_security title: Symantec Endpoint Security -version: "0.3.0" +version: "0.3.1" description: Collect logs from Symantec Endpoint Security with Elastic Agent. type: integration categories: