From d3dc3b64e42fe090543ce09a352cad670d11c3ac Mon Sep 17 00:00:00 2001 From: Thomas Walter Date: Thu, 8 Feb 2024 08:26:35 +0100 Subject: [PATCH] Rename patterns to correspond to field name --- 50-filter-postfix.conf | 4 +- postfix.grok | 54 +++++++++---------- .../{relay_info_0001.yaml => relay_0001.yaml} | 4 +- 3 files changed, 31 insertions(+), 31 deletions(-) rename test/{relay_info_0001.yaml => relay_0001.yaml} (74%) diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf index cb5932e..b48acdb 100644 --- a/50-filter-postfix.conf +++ b/50-filter-postfix.conf @@ -201,7 +201,7 @@ filter { if [postfix_client] { grok { patterns_dir => "/etc/logstash/patterns.d" - match => ["postfix_client", "^%{POSTFIX_CLIENT_INFO}$"] + match => ["postfix_client", "^%{POSTFIX_CLIENT}$"] tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ] remove_field => [ "postfix_client" ] } @@ -209,7 +209,7 @@ filter { if [postfix_relay] { grok { patterns_dir => "/etc/logstash/patterns.d" - match => ["postfix_relay", "^%{POSTFIX_RELAY_INFO}$"] + match => ["postfix_relay", "^%{POSTFIX_RELAY}$"] tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ] remove_field => [ "postfix_relay" ] } diff --git a/postfix.grok b/postfix.grok index 0fba659..b532f58 100644 --- a/postfix.grok +++ b/postfix.grok @@ -8,8 +8,8 @@ IP_UNKNOWN unknown # common postfix patterns POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}|NOQUEUE) -POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\](:%{INT:postfix_client_port})? -POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} +POSTFIX_CLIENT %{HOSTNAME:postfix_client_hostname}?\[(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\](:%{INT:postfix_client_port})? +POSTFIX_RELAY %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|BDAT|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.) POSTFIX_ACTION (accept|defer|discard|filter|header-redirect|reject|reject_warning) POSTFIX_STATUS_CODE \d{3} @@ -24,8 +24,8 @@ POSTFIX_WARNING_LEVEL (warning|fatal|info) POSTFIX_VERIFY_CLEANUP_TYPE (full|partial) -POSTFIX_TLSCONN %{DATA:postfix_tls_trustlevel} TLS connection established (to %{POSTFIX_RELAY_INFO}|from %{POSTFIX_CLIENT_INFO}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\)( key-exchange %{DATA:postfix_tls_key_exchange} server-signature %{DATA:postfix_tls_server_signature} \(%{DATA:postfix_tls_server_signature_size} bits\) server-digest %{DATA:postfix_tls_server_digest})? -POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY_INFO}: %{GREEDYDATA:postfix_tls_error} +POSTFIX_TLSCONN %{DATA:postfix_tls_trustlevel} TLS connection established (to %{POSTFIX_RELAY}|from %{POSTFIX_CLIENT}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\)( key-exchange %{DATA:postfix_tls_key_exchange} server-signature %{DATA:postfix_tls_server_signature} \(%{DATA:postfix_tls_server_signature_size} bits\) server-digest %{DATA:postfix_tls_server_digest})? +POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY}: %{GREEDYDATA:postfix_tls_error} POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission} POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1|Address not available|Operation timed out|Address not available|Operation timed out|bare received) @@ -35,22 +35,22 @@ POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:pos # warning patterns -POSTFIX_WARNING_WITH_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT_INFO}: )?%{GREEDYDATA:postfix_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} -POSTFIX_WARNING_WITHOUT_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT_INFO}: )?%{GREEDYDATA:postfix_message} +POSTFIX_WARNING_WITH_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT}: )?%{GREEDYDATA:postfix_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} +POSTFIX_WARNING_WITHOUT_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT}: )?%{GREEDYDATA:postfix_message} POSTFIX_WARNING %{POSTFIX_WARNING_WITH_KV}|%{POSTFIX_WARNING_WITHOUT_KV} # smtpd patterns -POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT_INFO} -POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT_INFO}( %{GREEDYDATA:postfix_command_counter_data})? -POSTFIX_SMTPD_LOSTCONN %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( \(%{INT} bytes\))?)? from %{POSTFIX_CLIENT_INFO}(: %{GREEDYDATA:postfix_smtpd_lostconn_reason})? -POSTFIX_SMTPD_NOQUEUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} -POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_improper_pipelining_data} +POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT} +POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT}( %{GREEDYDATA:postfix_command_counter_data})? +POSTFIX_SMTPD_LOSTCONN %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( \(%{INT} bytes\))?)? from %{POSTFIX_CLIENT}(: %{GREEDYDATA:postfix_smtpd_lostconn_reason})? +POSTFIX_SMTPD_NOQUEUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} +POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT}: %{GREEDYDATA:postfix_improper_pipelining_data} POSTFIX_SMTPD_PROXY proxy-%{POSTFIX_ACTION:postfix_proxy_result}: (%{POSTFIX_SMTP_STAGE:postfix_proxy_smtp_stage}): %{POSTFIX_PROXY_MESSAGE:postfix_proxy_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} # cleanup patterns POSTFIX_CLEANUP_MILTER %{POSTFIX_QUEUEID:postfix_queueid}: milter-%{POSTFIX_ACTION:postfix_milter_result}: %{GREEDYDATA:postfix_milter_message}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}(: %{GREEDYDATA:postfix_milter_data})? POSTFIX_CLEANUP_PREPEND_TYPE (header|body) -POSTFIX_CLEANUP_PREPEND %{POSTFIX_QUEUEID:postfix_queueid}: prepend: %{POSTFIX_CLEANUP_PREPEND_TYPE:postfix_prepend_type} %{GREEDYDATA:postfix_prepend_trigger} from %{POSTFIX_CLIENT_INFO}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}: %{GREEDYDATA:postfix_prepend_value} +POSTFIX_CLEANUP_PREPEND %{POSTFIX_QUEUEID:postfix_queueid}: prepend: %{POSTFIX_CLEANUP_PREPEND_TYPE:postfix_prepend_type} %{GREEDYDATA:postfix_prepend_trigger} from %{POSTFIX_CLIENT}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}: %{GREEDYDATA:postfix_prepend_value} POSTFIX_CLEANUP_MESSAGEID %{POSTFIX_QUEUEID:postfix_queueid}: message-id=? # qmgr patterns @@ -75,19 +75,19 @@ POSTFIX_POSTSUPER_SUMMARY_ACTIONS (Deleted|Requeued|Placed on hold|Released from POSTFIX_POSTSUPER_SUMMARY %{POSTFIX_POSTSUPER_SUMMARY_ACTIONS:postfix_postsuper_summary_action}: %{NUMBER:postfix_postsuper_summary_count} messages? # postscreen patterns -POSTFIX_PS_CONNECT CONNECT from %{POSTFIX_CLIENT_INFO} to \[%{IP:postfix_server_ip}\]:%{INT:postfix_server_port} -POSTFIX_PS_ACCESS %{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access} %{POSTFIX_CLIENT_INFO} +POSTFIX_PS_CONNECT CONNECT from %{POSTFIX_CLIENT} to \[%{IP:postfix_server_ip}\]:%{INT:postfix_server_port} +POSTFIX_PS_ACCESS %{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access} %{POSTFIX_CLIENT} POSTFIX_PS_NOQUEUE %{POSTFIX_SMTPD_NOQUEUE} -POSTFIX_PS_TOOBUSY NOQUEUE: reject: CONNECT from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_postscreen_toobusy_data} -POSTFIX_PS_DNSBL %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation} rank %{INT:postfix_postscreen_dnsbl_rank} for %{POSTFIX_CLIENT_INFO} +POSTFIX_PS_TOOBUSY NOQUEUE: reject: CONNECT from %{POSTFIX_CLIENT}: %{GREEDYDATA:postfix_postscreen_toobusy_data} +POSTFIX_PS_DNSBL %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation} rank %{INT:postfix_postscreen_dnsbl_rank} for %{POSTFIX_CLIENT} POSTFIX_PS_CACHE cache %{DATA} full cleanup: retained=%{NUMBER:postfix_postscreen_cache_retained} dropped=%{NUMBER:postfix_postscreen_cache_dropped} entries -POSTFIX_PS_VIOLATIONS %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}( %{INT})?( after %{NUMBER:postfix_postscreen_violation_time})? from %{POSTFIX_CLIENT_INFO}(( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})?(: %{GREEDYDATA:postfix_postscreen_data})?| in tests (after|before) SMTP handshake) +POSTFIX_PS_VIOLATIONS %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}( %{INT})?( after %{NUMBER:postfix_postscreen_violation_time})? from %{POSTFIX_CLIENT}(( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})?(: %{GREEDYDATA:postfix_postscreen_data})?| in tests (after|before) SMTP handshake) # dnsblog patterns POSTFIX_DNSBLOG_LISTING addr %{IP:postfix_client_ip} listed by domain %{HOSTNAME:postfix_dnsbl_domain} as %{IP:postfix_dnsbl_result} # tlsproxy patterns -POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT_INFO} +POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT} # anvil patterns POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_service}:(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} @@ -96,15 +96,15 @@ POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil # smtp patterns POSTFIX_SMTP_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_smtp_response}\))? -POSTFIX_SMTP_CONNERR connect to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} -POSTFIX_SMTP_SSLCONNERR SSL_connect error to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} -POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY_INFO}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? -POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY_INFO} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? -POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\) -POSTFIX_SMTP_RELAYREFUSED %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} refused to talk to me: %{GREEDYDATA:postfix_smtp_response} -POSTFIX_SMTP_SSLAUTHERR %{POSTFIX_QUEUEID:postfix_queueid}: SASL authentication failed; server %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} -POSTFIX_SMTP_UTF8 host %{POSTFIX_RELAY_INFO} offers SMTPUTF8 support, but not 8BITMIME -POSTFIX_SMTP_PIX %{POSTFIX_QUEUEID:postfix_queueid}: enabling PIX workarounds: %{DATA:postfix_pix_workaround} for %{POSTFIX_RELAY_INFO} +POSTFIX_SMTP_CONNERR connect to %{POSTFIX_RELAY}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} +POSTFIX_SMTP_SSLCONNERR SSL_connect error to %{POSTFIX_RELAY}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} +POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? +POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? +POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\) +POSTFIX_SMTP_RELAYREFUSED %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY} refused to talk to me: %{GREEDYDATA:postfix_smtp_response} +POSTFIX_SMTP_SSLAUTHERR %{POSTFIX_QUEUEID:postfix_queueid}: SASL authentication failed; server %{POSTFIX_RELAY} said: %{GREEDYDATA:postfix_smtp_response} +POSTFIX_SMTP_UTF8 host %{POSTFIX_RELAY} offers SMTPUTF8 support, but not 8BITMIME +POSTFIX_SMTP_PIX %{POSTFIX_QUEUEID:postfix_queueid}: enabling PIX workarounds: %{DATA:postfix_pix_workaround} for %{POSTFIX_RELAY} # master patterns POSTFIX_MASTER_START (daemon started|reload) -- version %{DATA:postfix_version}, configuration %{PATH:postfix_config_path} diff --git a/test/relay_info_0001.yaml b/test/relay_0001.yaml similarity index 74% rename from test/relay_info_0001.yaml rename to test/relay_0001.yaml index 926f5eb..7133938 100644 --- a/test/relay_info_0001.yaml +++ b/test/relay_0001.yaml @@ -1,5 +1,5 @@ -# this data is derived after the keyvalue data is expanded in logstash to a 'postfix_relay_info' field. -pattern: ^%{POSTFIX_RELAY_INFO}$ +# this data is derived after the keyvalue data is expanded in logstash to a 'postfix_relay' field. +pattern: ^%{POSTFIX_RELAY}$ data: "1.example.com.si[private/dovecot-lmtp]" results: postfix_relay_hostname: 1.example.com.si