From 7960af63372248642fd1bae15ce8c1cde0f919a3 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 28 Aug 2024 12:45:37 +0200 Subject: [PATCH] [filebeat] Document winlog input and make config on par with winlogbeat (#40462) * Document winlog input * Preallocate sources * Change to use a single event_log config per input * Remove changelog entry * Fix input doc links --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/filebeat-options.asciidoc | 4 + filebeat/docs/inputs/input-winlog.asciidoc | 459 ++++++++++++++++++ winlogbeat/eventlog/wineventlog.go | 4 +- .../eventlog/wineventlog_experimental.go | 4 +- 5 files changed, 470 insertions(+), 2 deletions(-) create mode 100644 filebeat/docs/inputs/input-winlog.asciidoc diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 21690cb77058..875c6768a6e8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -288,6 +288,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Update CEL mito extensions to v1.15.0. {pull}40294[40294] - Allow cross-region bucket configuration in s3 input. {issue}22161[22161] {pull}40309[40309] - Improve logging in Okta Entity Analytics provider. {issue}40106[40106] {pull}40347[40347] +- Document `winlog` input. {issue}40074[40074] {pull}40462[40462] - Added retry logic to websocket connections in the streaming input. {issue}40271[40271] {pull}40601[40601] *Auditbeat* diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index 860f98f2356a..3b1d274d59bd 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -95,6 +95,8 @@ You can configure {beatname_uc} to use the following inputs: * <<{beatname_lc}-input-syslog>> * <<{beatname_lc}-input-tcp>> * <<{beatname_lc}-input-udp>> +* <<{beatname_lc}-input-unix>> +* <<{beatname_lc}-input-winlog>> include::multiline.asciidoc[] @@ -157,3 +159,5 @@ include::inputs/input-tcp.asciidoc[] include::inputs/input-udp.asciidoc[] include::inputs/input-unix.asciidoc[] + +include::inputs/input-winlog.asciidoc[] diff --git a/filebeat/docs/inputs/input-winlog.asciidoc b/filebeat/docs/inputs/input-winlog.asciidoc new file mode 100644 index 000000000000..6d58c7d8df8d --- /dev/null +++ b/filebeat/docs/inputs/input-winlog.asciidoc @@ -0,0 +1,459 @@ +:type: winlog + +[id="{beatname_lc}-input-{type}"] +=== winlog input + +++++ +winlog +++++ + +beta[] + +Use the `winlog` input to read Windows event logs. It reads from one +event log using Windows APIs, filters the events based on user-configured criteria, +then sends the event data to the configured outputs. It watches the event +log so that new event data is sent in a timely manner. The read position for +the event log is persisted to disk to allow the input to resume after +restarts. + +The `winlog` input can capture event data from any event logs running on your system. +For example, you can capture events such as: + +* application events +* hardware events +* security events +* system events + +Here is a sample configuration: + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Application + ignore_older: 72h +-------------------------------------------------------------------------------- + +[float] +=== Configuration options + +[float] +==== `batch_read_size` + +The maximum number of event log records to read from the Windows API in a single +batch. The default batch size is 100. Most Windows versions return an error if +the value is larger than 1024. *{vista_and_newer}* + +{beatname_uc} starts a goroutine (a lightweight thread) to read from each +individual event log. The goroutine reads a batch of event log records using the +Windows API, applies any processors to the events, publishes them to the +configured outputs, and waits for an acknowledgement from the outputs before +reading additional event log records. + +[float] +==== `name` + +The name of the event log to monitor. It must +have a `name` field, except for those which use a custom XML query. +A channel is a named stream of events that transports events from an event +source to an event log. Most channels are tied to specific event publishers. +You can get a list of available event logs by using the PowerShell +https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.diagnostics/get-winevent[`Get-WinEvent`] cmdlet +on Windows Vista or newer. Here is a sample of the output from the command: + +[source,sh] +-------------------------------------------------------------------------------- +PS C:\> Get-WinEvent -ListLog * | Format-List -Property LogName +LogName : Application +LogName : HardwareEvents +LogName : Internet Explorer +LogName : Key Management Service +LogName : Security +LogName : System +LogName : Windows PowerShell +LogName : ForwardedEvents +LogName : Microsoft-Management-UI/Admin +LogName : Microsoft-Rdms-UI/Admin +LogName : Microsoft-Rdms-UI/Operational +LogName : Microsoft-Windows-Windows Firewall With Advanced Security/Firewall +... +-------------------------------------------------------------------------------- + +If `Get-WinEvent` is not available, the https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.management/get-eventlog[`Get-EventLog`] cmdlet can be used in its +place. + +[source,sh] +-------------------------------------------------------------------------------- +PS C:\Users\vagrant> Get-EventLog * + + Max(K) Retain OverflowAction Entries Log + ------ ------ -------------- ------- --- + 20,480 0 OverwriteAsNeeded 75 Application + 20,480 0 OverwriteAsNeeded 0 HardwareEvents + 512 7 OverwriteOlder 0 Internet Explorer + 20,480 0 OverwriteAsNeeded 0 Key Management Service + 20,480 0 OverwriteAsNeeded 1,609 Security + 20,480 0 OverwriteAsNeeded 1,184 System + 15,360 0 OverwriteAsNeeded 464 Windows PowerShell +-------------------------------------------------------------------------------- + +You must specify the full name of the channel in the configuration file. + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Microsoft-Windows-Windows Firewall With Advanced Security/Firewall +-------------------------------------------------------------------------------- + +To read events from an archived `.evtx` file you can specify the `name` as the +absolute path (it cannot be relative) to the file. + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: 'C:\backup\sysmon-2019.08.evtx' + no_more_events: stop +-------------------------------------------------------------------------------- + +The name key must not be used with custom XML queries. + +[float] +==== `id` + +A unique identifier for the event log. This key is required when using a custom +XML query. + +It is used to uniquely identify the event log reader in the registry file. This is +useful if multiple event logs are being set up to watch the same channel or file. If an +ID is not given, the `name` value will be used. + +This value must be unique. + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Application + id: application-logs + ignore_older: 168h +-------------------------------------------------------------------------------- + +[float] +==== `ignore_older` + +If this option is specified, the input filters events that are older than the +specified amount of time. Valid time units are "ns", "us" (or "µs"), "ms", "s", +"m", "h". This option is useful when you are beginning to monitor an event log +that contains older records that you would like to ignore. This field is +optional. + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Application + ignore_older: 168h +-------------------------------------------------------------------------------- + +[float] +==== `forwarded` + +A boolean flag to indicate that the log contains only events collected from +remote hosts using the Windows Event Collector. The value defaults to true for +the ForwardedEvents log and false for any other log. *{vista_and_newer}* + +This settings allows {beatname_uc} to optimize reads for forwarded events that are +already rendered. When the value is true {beatname_uc} does not attempt to render +the event using message files from the host computer. The Windows Event +Collector subscription should be configured to use the "RenderedText" format +(this is the default) to ensure that the events are distributed with messages +and descriptions. + +[float] +==== `event_id` + +An allowlist and blocklist of event IDs. The value is a comma-separated list. The +accepted values are single event IDs to include (e.g. 4624), a range of event +IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735). +*{vista_and_newer}* + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Security + event_id: 4624, 4625, 4700-4800, -4735 +-------------------------------------------------------------------------------- + +[WARNING] +======================================= +If you specify more than 22 query conditions (event IDs or event ID ranges), some +versions of Windows will prevent {beatname_uc} from reading the event log due to +limits in the query system. If this occurs a similar warning as shown below will +be logged by {beatname_uc}, and it will continue processing data from other event +logs. + +`WARN EventLog[Application] Open() error. No events will be read from this +source. The specified query is invalid.` + +In some cases, the limit may be lower than 22 conditions. For instance, using a +mixture of ranges and single event IDs, along with an additional parameter such +as `ignore older`, results in a limit of 21 conditions. + +If you have more than 22 conditions, you can workaround this Windows limitation +by using a drop_event[drop-event] processor to do the filtering after +{beatname_uc} has received the events from Windows. The filter shown below is +equivalent to `event_id: 903, 1024, 4624` but can be expanded beyond 22 +event IDs. + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Security + processors: + - drop_event.when.not.or: + - equals.winlog.event_id: 903 + - equals.winlog.event_id: 1024 + - equals.winlog.event_id: 4624 +-------------------------------------------------------------------------------- + +======================================= + +[float] +==== `language` + +The language ID the events will be rendered in. The language will be forced regardless +of the system language. A complete list of language IDs can be found +https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c[here]. +It defaults to `0`, which indicates to use the system language. + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Security + event_id: 4624, 4625, 4700-4800, -4735 + language: 0x0409 # en-US +-------------------------------------------------------------------------------- + +[float] +==== `level` + +A list of event levels to include. The value is a comma-separated list of +levels. *{vista_and_newer}* + +[cols="2*", options="header"] +|=== +|Level +|Value + +|critical, crit +|1 + +|error, err +|2 + +|warning, warn +|3 + +|information, info +|0 or 4 + +|verbose +|5 +|=== + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Security + level: critical, error, warning +-------------------------------------------------------------------------------- + +[float] +==== `provider` + +A list of providers (source names) to include. The value is a YAML list. +*{vista_and_newer}* + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Application + provider: + - Application Error + - Application Hang + - Windows Error Reporting + - EMET +-------------------------------------------------------------------------------- + +You can obtain a list of providers associated with a log by using PowerShell. +Here is an example showing the providers associated with the Security log. + +[source,sh] +-------------------------------------------------------------------------------- +PS C:\> (Get-WinEvent -ListLog Security).ProviderNames +DS +LSA +SC Manager +Security +Security Account Manager +ServiceModel 4.0.0.0 +Spooler +TCP/IP +VSSAudit +Microsoft-Windows-Security-Auditing +Microsoft-Windows-Eventlog +-------------------------------------------------------------------------------- + +[float] +==== `xml_query` + +Provide a custom XML query. This option is mutually exclusive with the `name`, `event_id`, +`ignore_older`, `level`, and `provider` options. These options should be included in +the XML query directly. Furthermore, an `id` must be provided. Custom XML queries +provide more flexibility and advanced options than the simpler query options in {beatname_uc}. +*{vista_and_newer}* + +Here is a configuration which will collect DHCP server events from multiple channels: + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + id: dhcp-server-logs + xml_query: > + + + + + + + +-------------------------------------------------------------------------------- + +XML queries may also be created in Windows Event Viewer using custom views. The query +can be created using a graphical interface and the corresponding XML can be +retrieved from the XML tab. + +[float] +==== `include_xml` + +Boolean option that controls if the raw XML representation of an event is +included in the data sent by {beatname_uc}. The default is false. +*{vista_and_newer}* + +The XML representation of the event is useful for troubleshooting purposes. The +data in the fields reported by {beatname_uc} can be compared to the data in the XML +to diagnose problems. + +Example: + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: Microsoft-Windows-Windows Defender/Operational + include_xml: true +-------------------------------------------------------------------------------- + +[float] +==== `tags` + +A list of tags that the Beat includes in the `tags` field of each published +event. Tags make it easy to select specific events in Kibana or apply +conditional filtering in Logstash. These tags will be appended to the list of +tags specified in the general configuration. + +Example: + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: CustomLog + tags: ["web"] +-------------------------------------------------------------------------------- + +[float] +[[winlog-configuration-fields]] +==== `fields` + +Optional fields that you can specify to add additional information to the +output. For example, you might add fields that you can use for filtering event +data. Fields can be scalar values, arrays, dictionaries, or any nested +combination of these. By default, the fields that you specify here will be +grouped under a `fields` sub-dictionary in the output document. To store the +custom fields as top-level fields, set the `fields_under_root` option to true. +If a duplicate field is declared in the general configuration, then its value +will be overwritten by the value declared here. + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: CustomLog + fields: + customer_id: 51415432 +-------------------------------------------------------------------------------- + +[float] +==== `fields_under_root` + +If this option is set to true, the custom <> +are stored as top-level fields in the output document instead of being grouped +under a `fields` sub-dictionary. If the custom field names conflict with other +field names added by {beatname_uc}, then the custom fields overwrite the other +fields. + +[float] +==== `processors` + +A list of processors to apply to the data generated by the event log. + +See <> for information about specifying +processors in your config. + +[float] +==== `index` + +If present, this formatted string overrides the index for events from this +event log (for elasticsearch outputs), or sets the `raw_index` field of the event's +metadata (for other outputs). This string can only refer to the agent name and +version and the event timestamp; for access to dynamic fields, use +`output.elasticsearch.index` or a processor. + +Example value: `"%{[agent.name]}-myindex-%{+yyyy.MM.dd}"` might +expand to `"filebeat-myindex-2019.12.13"`. + +[float] +==== `keep_null` + +If this option is set to true, fields with `null` values will be published in +the output document. By default, `keep_null` is set to `false`. + +[float] +==== `no_more_events` + +The action that the event log reader should take when it receives a signal from +Windows that there are no more events to read. It can either `wait` for more +events to be written (the default behavior) or it can `stop`. The overall +{beatname_uc} process will stop when all of the individual event log readers have +stopped. *{vista_and_newer}* + +Setting `no_more_events` to `stop` is useful when reading from archived event +log files where you want to read the whole file then exit. + +[float] +==== `api` + +This selects the event log reader implementation that is used to read events +from the Windows APIs. You should only set this option when testing experimental +features. When the value is set to `wineventlog-experimental` {beatname_uc} will +replace the default event log reader with the **experimental** implementation. +We are evaluating this implementation to see if it can provide increased +performance and reduce CPU usage. *{vista_and_newer}* + +[source,yaml] +-------------------------------------------------------------------------------- +- type: winlog + name: ForwardedEvents + api: wineventlog-experimental +-------------------------------------------------------------------------------- + +There are a few notable differences in the events: + +* Events that contained data under `winlog.user_data` will now have it under + `winlog.event_data`. +* Setting `include_xml: true` has no effect. diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index ddb703dbbb08..d558a477845e 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -270,7 +270,6 @@ func newWinEventLog(options *conf.C) (EventLog, error) { cache: newMessageFilesCache(id, eventMetadataHandle, freeHandle), winMetaCache: newWinMetaCache(metaTTL), logPrefix: fmt.Sprintf("WinEventLog[%s]", id), - metrics: newInputMetrics(c.Name, id), } // Forwarded events should be rendered using RenderEventXML. It is more @@ -316,6 +315,9 @@ func (l *winEventLog) IsFile() bool { func (l *winEventLog) Open(state checkpoint.EventLogState) error { var bookmark win.EvtHandle var err error + // we need to defer metrics initialization since when the event log + // is used from winlog input it would register it twice due to CheckConfig calls + l.metrics = newInputMetrics(l.channelName, l.id) if len(state.Bookmark) > 0 { bookmark, err = win.CreateBookmarkFromXML(state.Bookmark) } else if state.RecordNumber > 0 && l.channelName != "" { diff --git a/winlogbeat/eventlog/wineventlog_experimental.go b/winlogbeat/eventlog/wineventlog_experimental.go index 5e323858f229..3a6a9b2fe92b 100644 --- a/winlogbeat/eventlog/wineventlog_experimental.go +++ b/winlogbeat/eventlog/wineventlog_experimental.go @@ -129,7 +129,6 @@ func newWinEventLogExp(options *conf.C) (EventLog, error) { maxRead: c.BatchReadSize, renderer: renderer, log: log, - metrics: newInputMetrics(c.Name, id), } return l, nil @@ -157,6 +156,9 @@ func (l *winEventLogExp) IsFile() bool { func (l *winEventLogExp) Open(state checkpoint.EventLogState) error { l.lastRead = state + // we need to defer metrics initialization since when the event log + // is used from winlog input it would register it twice due to CheckConfig calls + l.metrics = newInputMetrics(l.channelName, l.id) var err error l.iterator, err = win.NewEventIterator(