Merge pull request #1531 from dmwm/logstash_xps_update

Update x509-proxy-server.yaml
dmwm · Aug 6, 2024 · 3ee4180 · 3ee4180
2 parents b632444 + ccd9fb8
commit 3ee4180
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 8 deletions.
diff --git a/kubernetes/cmsweb/daemonset/x509-proxy-server.yaml b/kubernetes/cmsweb/daemonset/x509-proxy-server.yaml
@@ -17,10 +17,13 @@ data:
       scan_frequency: 10s
       backoff: 5s
       max_backoff: 10s
-    output.console:
-      codec.format:
-        string: '%{[message]} - Podname=${MY_POD_NAME}'
-        pretty: false
+      tags: ["aps"]
+    output.logstash:
+      hosts: ["logstash.monitoring:5044"]
+      compression_level: 3
+      worker: 4
+      bulk_max_size: 4096
+      pipelining: 2
     queue.mem:
       events: 65536
     logging.metrics.enabled: false

diff --git a/kubernetes/cmsweb/monitoring/logstash.conf b/kubernetes/cmsweb/monitoring/logstash.conf
@@ -58,7 +58,6 @@ filter {
       }
       if ![api] {
           mutate { replace => { "api" => "%{request}" } }
-          mutate { replace => { "system" => "%{request}" } }
       }
       if [client] {
           grok { match => { "client" => '%{DATA:client_name}/%{DATA:client_version}$' } }
@@ -81,10 +80,14 @@ filter {
       grok {
         match => { "message" => '\[%{TIMESTAMP_ISO8601:tstamp}\] %{DATA:httpversion} %{NUMBER:code:int} %{WORD:method} %{NOTSPACE:request} \[data: %{NUMBER:bytes_received:int} in %{NUMBER:bytes_sent:int} out\] \[host: %{IPORHOST:frontend}:%{NUMBER:fe_port}\] \[remoteAddr: %{IPORHOST:clientip}:%{NUMBER:clientport:int}\] \[X-Forwarded-For: %{IPORHOST:x_forwarded_ip}:%{NUMBER:x_forwarded_port:int}\] \[X-Forwarded-Host: %{HOSTNAME:x_forwarded_host}\] \[auth: %{DATA:tls} %{DATA:crypto} "%{DATA:dn}" %{DATA:auth_name} %{WORD:auth_protocol}\] \[ref: "%{DATA:cluster}" "%{DATA:client}"\] \[req: %{NUMBER:request_time:float} \(s\) proxy-resp: %{NUMBER:proxy_resp_time:float} \(s\)\]' }
       }
-
       grok {
-      pattern_definitions => { "WORDHYPHEN" => "\b[\w\-]+\b" }
-      match => { "request" => '/%{WORDHYPHEN:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' }
+        match => {
+          "cmsweb_log" => "/data/srv/logs/frontend/%{DATA:log_prefix}_%{GREEDYDATA:proxy_server}-%{DATA}_%{YEAR}%{MONTHNUM}%{MONTHDAY}"
+        }
+      }
+      grok {
+        pattern_definitions => { "WORDHYPHEN" => "\b[\w\-]+\b" }
+        match => { "request" => '/%{WORDHYPHEN:system}%{UNIXPATH:uri_path}%{URIPARAM:uri_params}?' }
       }
       if [system] =~ /^(wmstatsserver|reqmgr2|t0_reqmon|ms-pileup|ms-transferor|ms-monitor|ms-output|ms-unmerged|ms-rulecleaner)$/ {
           grok {