From fcf324c3cca881d831cd652a07eef2f3909a8e7e Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:41:30 -0600 Subject: [PATCH] [cisco_asa] Add advanced option for time zone mapping and support parsing extra timestamp in header (#12440) - Added an advanced option for mapping short time zone names to long time zone names - Support parsing the extra timestamp that is sometimes included in messages, often containing the time zone. --- packages/cisco_asa/changelog.yml | 5 + ...test-additional-messages.log-expected.json | 4 +- .../test/pipeline/test-asa-tz-mapping.log | 2 + .../test-asa-tz-mapping.log-config.yml | 11 ++ .../test-asa-tz-mapping.log-expected.json | 143 ++++++++++++++++++ .../_dev/test/pipeline/test-common-config.yml | 6 +- .../test-invalid-data.log-expected.json | 9 +- .../elasticsearch/ingest_pipeline/default.yml | 27 ++++ .../cisco_asa/data_stream/log/manifest.yml | 36 ++++- packages/cisco_asa/manifest.yml | 2 +- 10 files changed, 232 insertions(+), 13 deletions(-) create mode 100644 packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log create mode 100644 packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-config.yml create mode 100644 packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-expected.json diff --git a/packages/cisco_asa/changelog.yml b/packages/cisco_asa/changelog.yml index decd40709f1..7fb93e51ddf 100644 --- a/packages/cisco_asa/changelog.yml +++ b/packages/cisco_asa/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.41.0" + changes: + - description: "Add advanced option for time zone mapping and support parsing extra timestamp in header." + type: enhancement + link: https://github.com/elastic/integrations/pull/12440 - version: "2.40.0" changes: - description: "Set event.outcome to failure for all denied events." diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-additional-messages.log-expected.json b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-additional-messages.log-expected.json index 96df13da38f..f1e32b0a108 100644 --- a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-additional-messages.log-expected.json +++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-additional-messages.log-expected.json @@ -11566,7 +11566,6 @@ "original": "Sep 25 15:47:07 host1.example.com : Sep 25 15:47:07 EDT: %ASA-auth-4-113004: AAA user authentication Successful : server = myservername : user = myusername", "outcome": "success", "severity": 4, - "timezone": "UTC", "type": [ "allowed", "info" @@ -11628,7 +11627,6 @@ "original": "Sep 25 12:42:21 host1.example.com : Sep 25 12:42:21 EDT: %ASA-auth-4-113005: AAA user authentication Rejected : reason = AAA failure : server = myservername : user = myusername : user IP = 10.11.74.55", "outcome": "failure", "severity": 4, - "timezone": "UTC", "type": [ "denied", "info" @@ -11670,7 +11668,7 @@ ] }, { - "@timestamp": "2025-09-25T01:08:29.000Z", + "@timestamp": "2025-09-25T05:08:29.000Z", "cisco": { "asa": { "destination_interface": "GWAN", diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log new file mode 100644 index 00000000000..abf9a07244c --- /dev/null +++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log @@ -0,0 +1,2 @@ +Sep 25 15:47:07 host1.example.com : Sep 25 15:47:07 EDT: %ASA-auth-4-113004: AAA user authentication Successful : server = myservername : user = myusername +Jan 22 14:05:11 test.example.com : Jan 22 14:05:11 PST: %ASA-svc-4-722051: Group User IP <81.2.69.144> IPv4 Address <10.20.0.1> IPv6 address <::> assigned to session diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-config.yml b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-config.yml new file mode 100644 index 00000000000..44877e6ba98 --- /dev/null +++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-config.yml @@ -0,0 +1,11 @@ +dynamic_fields: + "event.ingested": ".*" +fields: + tags: + - preserve_original_event + _conf: + tz_map: + - tz_short: EDT + tz_long: America/New_York + - tz_short: PST + tz_long: -08:00 diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-expected.json b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-expected.json new file mode 100644 index 00000000000..3b87cd8a444 --- /dev/null +++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-asa-tz-mapping.log-expected.json @@ -0,0 +1,143 @@ +{ + "expected": [ + { + "@timestamp": "2025-09-25T15:47:07.000-04:00", + "cisco": { + "asa": { + "aaa_type": "authentication", + "suffix": "auth" + } + }, + "destination": { + "address": "myservername", + "domain": "myservername" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "logged-in", + "category": [ + "authentication", + "network" + ], + "code": "113004", + "kind": "event", + "original": "Sep 25 15:47:07 host1.example.com : Sep 25 15:47:07 EDT: %ASA-auth-4-113004: AAA user authentication Successful : server = myservername : user = myusername", + "outcome": "success", + "severity": 4, + "timezone": "America/New_York", + "type": [ + "allowed", + "info" + ] + }, + "host": { + "hostname": "host1.example.com" + }, + "log": { + "level": "warning" + }, + "observer": { + "hostname": "host1.example.com", + "product": "asa", + "type": "firewall", + "vendor": "Cisco" + }, + "related": { + "hosts": [ + "host1.example.com", + "myservername" + ], + "user": [ + "myusername" + ] + }, + "source": { + "user": { + "name": "myusername" + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2025-01-22T14:05:11.000-08:00", + "cisco": { + "asa": { + "assigned_ip": "10.20.0.1", + "suffix": "svc" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "address-assigned", + "category": [ + "network" + ], + "code": "722051", + "kind": "event", + "original": "Jan 22 14:05:11 test.example.com : Jan 22 14:05:11 PST: %ASA-svc-4-722051: Group User IP <81.2.69.144> IPv4 Address <10.20.0.1> IPv6 address <::> assigned to session", + "outcome": "success", + "reason": "IPv4 Address <10.20.0.1> IPv6 address <::> assigned to session", + "severity": 4, + "timezone": "-08:00", + "type": [ + "connection", + "info" + ] + }, + "host": { + "hostname": "test.example.com" + }, + "log": { + "level": "warning" + }, + "observer": { + "hostname": "test.example.com", + "product": "asa", + "type": "firewall", + "vendor": "Cisco" + }, + "related": { + "hosts": [ + "test.example.com" + ], + "ip": [ + "81.2.69.144" + ], + "user": [ + "user_NAME" + ] + }, + "source": { + "address": "81.2.69.144", + "geo": { + "city_name": "London", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.5142, + "lon": -0.0931 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "81.2.69.144", + "user": { + "group": { + "name": "GroupPolicy_NAME" + }, + "name": "user_NAME" + } + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-common-config.yml b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-common-config.yml index a0742f1e378..69c9585583d 100644 --- a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-common-config.yml +++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-common-config.yml @@ -1,7 +1,7 @@ dynamic_fields: - "event.end": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$" - "event.start": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$" - "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$" + "event.end": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}(?:Z|[-+][0-9]{2}:[0-9]{2})$" + "event.start": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}(?:Z|[-+][0-9]{2}:[0-9]{2})$" + "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}(?:Z|[-+][0-9]{2}:[0-9]{2})$" fields: tags: - preserve_original_event diff --git a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-invalid-data.log-expected.json b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-invalid-data.log-expected.json index 3bb3583da04..cfef2f1dbdd 100644 --- a/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-invalid-data.log-expected.json +++ b/packages/cisco_asa/data_stream/log/_dev/test/pipeline/test-invalid-data.log-expected.json @@ -314,6 +314,7 @@ ] }, { + "@timestamp": "2023-07-14T08:23:43.398Z", "cisco": { "asa": { "message_id": "" @@ -359,6 +360,7 @@ ] }, { + "@timestamp": "2023-07-14T08:23:43.398Z", "cisco": { "asa": { "message_id": "" @@ -533,6 +535,7 @@ ] }, { + "@timestamp": "2023-07-14T08:23:43.398Z", "cisco": { "asa": { "message_id": "" @@ -659,7 +662,7 @@ ] }, { - "@timestamp": "2025-07-13T08:23:43.000Z", + "@timestamp": "2023-07-14T08:23:43.000Z", "cisco": { "asa": { "message_id": "" @@ -711,7 +714,7 @@ ] }, { - "@timestamp": "2025-07-13T08:23:43.000Z", + "@timestamp": "2023-07-14T08:23:43.000Z", "cisco": { "asa": { "message_id": "" @@ -754,7 +757,7 @@ ] }, { - "@timestamp": "2025-07-13T08:23:43.000Z", + "@timestamp": "2023-07-14T08:23:43.000Z", "cisco": { "asa": { "message_id": "" diff --git a/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 4277f80d51e..a9c2ae9cf0d 100644 --- a/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cisco_asa/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -41,6 +41,15 @@ processors: # exactly match the syntax for firepower management logs PROCESS_HOST: "(?:%{PROCESS:process.name}:\\s%{SYSLOGHOST:host.name})" HOST_PROCESS: "(?:%{SYSLOGHOST:host.hostname}:?\\s+)?(?:%{PROCESS:process.name}?(?:\\[%{POSINT:process.pid:long}\\])?)?" + - grok: + field: _temp_.full_message + tag: grok_extra_timestamp + if: ctx._temp_.full_message != null + patterns: + - '^%{ASA_DATE:_temp_.raw_date}: %{DATA:_temp_.full_message}$' + - '%{GREEDYDATA:_temp_.full_message}' + pattern_definitions: + ASA_DATE: "(?:%{DAY} )?%{MONTH} *%{MONTHDAY}(?: %{YEAR})? %{TIME}(?: %{TZ:_temp_.tz})?" - script: lang: painless tag: script_log_syslog @@ -90,6 +99,18 @@ processors: value: 7 if: "ctx?.event?.severity == null" + - script: + lang: painless + tag: script_tz_mapping + if: ctx._temp_?.tz != null && ctx._temp_?.tz != '' && ctx._conf?.tz_map != null + source: >- + for (def item : ctx._conf.tz_map) { + if (item.tz_short == ctx._temp_.tz) { + ctx._temp_.tz = item.tz_long; + break; + } + } + # Time zone can come from three sources, choose in order: log, config, locale, default to UTC. - set: field: _temp_.tz @@ -117,6 +138,7 @@ processors: # - date: if: ctx._temp_?.raw_date != null + tag: parse_raw_date timezone: "{{{ event.timezone }}}" field: "_temp_.raw_date" formats: @@ -135,6 +157,8 @@ processors: - "EEE MMM dd yyyy HH:mm:ss" - "MMM d yyyy HH:mm:ss z" - "MMM dd yyyy HH:mm:ss z" + - "MMM d yyyy HH:mm:ss.SSS z" + - "MMM dd yyyy HH:mm:ss.SSS z" - "EEE MMM d yyyy HH:mm:ss z" - "EEE MMM dd yyyy HH:mm:ss z" on_failure: @@ -144,6 +168,7 @@ processors: ignore_missing: true - date: if: ctx._temp_?.raw_date != null + tag: "parse_raw_date_fallback" field: "_temp_.raw_date" formats: - "ISO8601" @@ -161,6 +186,8 @@ processors: - "EEE MMM dd yyyy HH:mm:ss" - "MMM d yyyy HH:mm:ss z" - "MMM dd yyyy HH:mm:ss z" + - "MMM d yyyy HH:mm:ss.SSS z" + - "MMM dd yyyy HH:mm:ss.SSS z" - "EEE MMM d yyyy HH:mm:ss z" - "EEE MMM dd yyyy HH:mm:ss z" diff --git a/packages/cisco_asa/data_stream/log/manifest.yml b/packages/cisco_asa/data_stream/log/manifest.yml index 03b42acde2a..66a24287e0c 100644 --- a/packages/cisco_asa/data_stream/log/manifest.yml +++ b/packages/cisco_asa/data_stream/log/manifest.yml @@ -81,12 +81,22 @@ streams: - name: tz_offset type: text - title: Timezone + title: Default Time Zone multi: false required: false show_user: false default: UTC description: IANA time zone or time offset (e.g. `+0200`) to use when interpreting syslog timestamps without a time zone. + - name: tz_map + type: yaml + title: Time Zone Map + multi: false + required: false + show_user: false + description: A combination of time zones as they appear in the Cisco ASA log, in combination with a proper IANA time zone or offset (for example, Australia/Sydney or +10:00). + default: | + #- tz_short: AEST + # tz_long: Australia/Sydney - input: tcp title: Cisco ASA logs description: Collect Cisco ASA logs @@ -177,12 +187,22 @@ streams: description: Specify custom configuration options for the TCP input. See [TCP](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-tcp.html) for details. - name: tz_offset type: text - title: Timezone + title: Default Time Zone multi: false required: false show_user: false default: UTC description: IANA time zone or time offset (e.g. `+0200`) to use when interpreting syslog timestamps without a time zone. + - name: tz_map + type: yaml + title: Time Zone Map + multi: false + required: false + show_user: false + description: A combination of time zones as they appear in the Cisco ASA log, in combination with a proper IANA time zone or offset (for example, Australia/Sydney or +10:00). + default: | + #- tz_short: AEST + # tz_long: Australia/Sydney - input: logfile enabled: false title: Cisco ASA logs @@ -247,9 +267,19 @@ streams: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. - name: tz_offset type: text - title: Timezone + title: Default Time Zone multi: false required: false show_user: false default: UTC description: IANA time zone or time offset (e.g. `+0200`) to use when interpreting syslog timestamps without a time zone. + - name: tz_map + type: yaml + title: Time Zone Map + multi: false + required: false + show_user: false + description: A combination of time zones as they appear in the Cisco ASA log, in combination with a proper IANA time zone or offset (for example, Australia/Sydney or +10:00). + default: | + #- tz_short: AEST + # tz_long: Australia/Sydney diff --git a/packages/cisco_asa/manifest.yml b/packages/cisco_asa/manifest.yml index 8b6f0f47683..c0c8c1adb27 100644 --- a/packages/cisco_asa/manifest.yml +++ b/packages/cisco_asa/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: cisco_asa title: Cisco ASA -version: "2.40.0" +version: "2.41.0" description: Collect logs from Cisco ASA with Elastic Agent. type: integration categories: