logstash.conf

###################### Start of INPUT data ################################

######################
# Collect Activities #
######################
input {
  exec {
    command => 'cd /opt/activities&&java -jar /opt/activities/SQLTool.jar activities'
    interval => 30
  }

  exec {
    command => 'find /data/logs/activities/activities-*log -maxdepth 1 -type f -mtime +3 -exec rm {} \;'
    interval => 3600
  }
}


###############
# Redis input #
###############
input {
  redis {
    host => "localhost"
    type => "redis-input"
    data_type => "list"
    key => "logstash"
    codec => "json"
    #batch_count => 1
    #threads => 1
  }
}

################
# alfresco.log #
################
input {
  file {
    codec => multiline {
      pattern => "(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)"
      negate => true
      what => "previous"
    }
    type => "alfrescoLog"
    path => [ "/data/logs/**/alfresco*.log*" ]
    tags => 'alfrescoLog'
  }
}

#############
# share.log #
#############
input {
  file {
    codec => multiline {
      pattern => "(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)"
      negate => true
      what => "previous"
    }
    type => "shareLog"
    path => [ "/data/logs/**/share*.log*" ]
    tags => 'shareLog'
  }
}

############
# solr.log #
############
input {
  file {
    codec => multiline {
      pattern => "(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d)"
      negate => true
      what => "previous"
    }
    type => "solrLog"
    path => [ "/data/logs/**/solr*.log*" ]
    tags => 'solrLog'
  }
}

#####################
# Tomcat Access Log #
#####################
input {
  file {
    type => "TomcatAccessLog"
    path => "/data/logs/**/access-*.log*"
  }
}

#####################
# dstat performance #
#####################
input {
  file {
    type => "dstatPerformance"
    path => "/data/logs/**/dstat*"
  }
}

#######################
# Tomcat Catalina Log #
#######################
input {
  file {
    type => "alfrescoLog"
    path => ["/data/logs/**/catalina.out", "/data/logs/**/catalina*out*"]
  }
}

#################
# RM Audit logs #
#################
input {
  file {
    type => "auditRM"
    path => ["/data/logs/**/parsedAuditRM*log"]
  }
}

######
# GC #
######
input {
  file {
    type => "jstatbeat"
    path => ["/data/logs/**/jstatbeat*"]
    codec => "json"
  }
}

##############
# Packetbeat #
##############
input {
  file {
    #type => "packetbeat"
    path => ["/data/logs/**/packetbeat*"]
    codec => "json"
  }
}

##############
# Metricbeat #
##############
input {
  file {
    type => "metricsets"
    path => ["/data/logs/**/metricbeat*"]
    codec => "json"
  }
}

#########
# beats #
#########
input {
  beats {
    host => "0.0.0.0"
    port => "5044"
    type => "gc"
  }
}

#################
# Activities    #
#################
input {
  file {
    type => "activities"
    path => ["/data/logs/**/activities-*log"]
  }
}

#################
# Workflows     #
#################
input {
  file {
    type => "workflows"
    path => ["/data/logs/**/workflows-*log"]
  }
}

#################
# p6spy         #
#################
input {
  file {
    type => "p6spy"
    path => ["/data/logs/**/spy.log*"]
  }
}

####################
# ApacheAccessLogs #
####################
input {
  file {
    type => "ApacheAccessLog"
    path => ["/data/logs/**/*Apache_access.log*"]
  }
}

#################
# DB-HelthCheck #
#################
input {
  file {
    type => "DB-Health-Check"
    path => ["/data/logs/**/db-healthcheck-*.log*"]
  }
}

####################### End of INPUT data ###############################

####################################################################################################
# Here we start parsing the incoming messages and create fields we can use for reporting in Kibana #
####################################################################################################

################
# alfresco.log #
################

filter {
  if [type] == "alfrescoLog" {
    # replace double blank space with single blank space
    mutate {
      gsub => [
        "message", "  ", " "
      ]
    }

    # Drop empty lines
    if [message] =~ /^[\s]*$/
    {
      drop {}
    }

    grok {
      match => [ "message", "(^%{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM))" ]
      add_tag => ["tomcat-msg"]
    }

    grok {
      match => [ "message", "(^%{LOGLEVEL}: %{GREEDYDATA})" ]
      add_tag => ["tomcat-msg"]
    }

    if "tomcat-msg" in [tags] {
      drop {}
    }

    # Match incoming log entries to fields
    grok { 
      break_on_match => true
      keep_empty_captures => false
      # Solr Searches
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\]) Got: %{NOTSPACE} in %{NOTSPACE:responseTime} ms"]

      # Document Transformations
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\])\s*%{NUMBER:id}\s*%{WORD:from}\s*%{WORD:to}\s*%{WORD:logType}\s*%{DATA:file} %{NUMBER:size:double} (?<sizeType>KB|MB) %{DATA:subMsg} %{NOTSPACE:responseTime} (?<timeType>ms)"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\])\s*%{NUMBER:id}\s*%{WORD:from}\s*%{WORD:to}\s*%{WORD:logType}\s*%{DATA:file} %{NUMBER:size:double} (?<sizeType>KB|MB|bytes) %{NOTSPACE:responseTime} (?<timeType>ms)"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\])\s*%{NUMBER:id}\s*%{WORD:from}\s*%{WORD:to}\s*%{WORD:logType}\s*%{NUMBER:size:double} %{WORD:sizeType} %{GREEDYDATA:subMsg}"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\])\s*%{NUMBER:id}\s*%{WORD:from}\s*%{WORD:to}\s*%{WORD:logType}\s*%{NUMBER:size:double} %{WORD:sizeType} %{GREEDYDATA:subMsg}"]

      # Script execution and rendering
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\]) Executed script %{NOTSPACE:script} in %{NOTSPACE:responseTime}ms"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\]) Rendered template %{NOTSPACE:script} in %{NOTSPACE:responseTime}ms"]

      # RM entries
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\]) Found %{NUMBER:dispositionActions} disposition action definitions updates awaiting publishing." ]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\]) Processing %{NUMBER:rmNodes} (node|nodes)" ]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\])\s*- duration %{NUMBER:publishUpdateDuration}" ]

      ############################################## Repeating the same without the thread in case they are using Support Tools ###########################################
      # Solr Searches
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) Got: %{NOTSPACE:entries} in %{NOTSPACE:responseTime} ms"]

      # Document Transformations
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*%{NUMBER:id}\s*%{WORD:from}\s*%{WORD:to}\s*%{WORD:logType}\s*%{DATA:file} %{NUMBER:size:double} (?<sizeType>KB|MB) %{DATA:subMsg} %{NOTSPACE:responseTime} (?<timeType>ms)"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*%{NUMBER:id}\s*%{WORD:from}\s*%{WORD:to}\s*%{WORD:logType}\s*%{DATA:file} %{NUMBER:size:double} (?<sizeType>KB|MB|bytes) %{NOTSPACE:responseTime} (?<timeType>ms)"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*%{NUMBER:id}\s*%{WORD:from}\s*%{WORD:to}\s*%{WORD:logType}\s*%{DATA:file} %{NUMBER:size:double} %{WORD:sizeType} %{NOTSPACE:responseTime} %{WORD:timeType} %{GREEDYDATA:subMsg}"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*%{NUMBER:id} %{WORD:from}\s*%{WORD:to} \s*%{WORD:logType}\s*%{NUMBER:size:double} %{WORD:sizeType} %{GREEDYDATA:subMsg}"]


      # Script execution and rendering
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) Executed script %{NOTSPACE:script} in %{NOTSPACE:responseTime}ms"]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) Rendered template %{NOTSPACE:script} in %{NOTSPACE:responseTime}ms"]

      # RM entries
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) Found %{NUMBER:dispositionActions} disposition action definitions updates awaiting publishing." ]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) Processing %{NUMBER:rmNodes} (node|nodes)" ]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*- duration %{NUMBER:publishUpdateDuration}" ]

      # Catch all entry
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) (\[%{NOTSPACE:thread}\]) %{GREEDYDATA:Msg}" ]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\]) %{GREEDYDATA:data}" ]      
    }

    grok { 
      # Catch java exceptions
      match => [ "message", "(?m)Caused by:%{DATA:exceptionClass}: %{GREEDYDATA}" ] 
    }

    grok { 
      # Catch all for Msg
      match => [ "message", "%{TIMESTAMP_ISO8601:data}\s*%{LOGLEVEL:data}\s*%{NOTSPACE:data} %{GREEDYDATA:Msg}" ] 
    }

    grok {
      match => [ "message", "ScheduledDispositionJob" ]
      add_tag => [ "ScheduledDispositionJob" ]
    }

    grok {
      match => [ "message", "DispositionLifecycleJobExecuter" ]
      add_tag => [ "DispositionLifecycleJobExecuter"]
    }

    grok {
      match => [ "message", "NotifyOfRecordsDueForReviewJobExecuter" ]
      add_tag => [ "NotifyOfRecordsDueForReviewJobExecuter"]
    }

    grok {
      match => [ "message", "PublishUpdatesJobExecuter" ]
      add_tag => [ "PublishUpdatesJobExecuter"]
    }

    mutate {
      gsub => [ "duration", ",", "" ]
    }

    mutate {
      convert => [ "publishUpdateDuration", "float" ]
    }

    mutate {
      convert => [ "publishUpdateDuration", "float" ]
      convert => [ "rmNodes", "float" ]
      convert => [ "dispositionActions", "float" ]
      convert => [ "duration", "float" ]
      convert => [ "responseTime", "float" ]
    }

    mutate {
      gsub => [ "responseTime", ",", "" ]
    }

    date {
       match => ["logdate" , "yyyy-MM-dd HH:mm:ss,SSS"]
       target => "@timestamp"
    }
  }
}

#############
# share.log #
#############

filter {
  if [type] == "shareLog" {
    # replace double blank space with single blank space
    mutate {
      gsub => [
        "message", "  ", " "
      ]
    }

    grok {
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*(\[%{NOTSPACE:thread}\]) %{GREEDYDATA:Msg}" ]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*%{GREEDYDATA:Msg}" ]
      add_tag => [ "%{logLevel}" ]
    }

    date {
       match => ["logdate" , "yyyy-MM-dd HH:mm:ss,SSS"]
       target => "@timestamp"
    }

    grok {
      match => [ "message", "Exception" ]
      add_tag => [ "Exception"]
    }

    grok {
      match => [ "message", "NullPointerException" ]
      add_tag => [ "NullPointerException"]
    }

    grok {
      match => [ "message", "OutOfMemoryError" ]
      add_tag => [ "OutOfMemoryError"]
    }

    grok {
      match => [ "message", "Too many open files" ]
      add_tag => [ "FileHandles"]
    }
  }
}


############
# solr.log #
############

filter {
  if [type] == "solrLog" {
    # replace double blank space with single blank space
    mutate {
      gsub => [
      "message", "  ", " "
      ]
    }

    grok {
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*(\[%{NOTSPACE:thread}\]) %{GREEDYDATA:Msg}" ]
      match => [ "message", "%{TIMESTAMP_ISO8601:logdate}\s*%{LOGLEVEL:logLevel}\s*(\[%{NOTSPACE:class}\])\s*%{GREEDYDATA:Msg}" ]
      add_tag => [ "%{logLevel}" ]
    }

    date {
       match => ["logdate" , "yyyy-MM-dd HH:mm:ss,SSS"]
       target => "@timestamp"
    }

    grok {
      match => [ "message", "Exception" ]
      add_tag => [ "Exception"]
    }

    grok {
      match => [ "message", "NullPointerException" ]
      add_tag => [ "NullPointerException"]
    }

    grok {
      match => [ "message", "OutOfMemoryError" ]
      add_tag => [ "OutOfMemoryError"]
    }

    grok {
      match => [ "message", "Too many open files" ]
      add_tag => [ "FileHandles"]
    }
  }
}

####################
# ApacheAccessLogs #
####################
filter {
  if [type] == "ApacheAccessLog" {

    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
    date {
      match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    #geoip {
    #  source => "clientip"
    #  target => "geoip"
    #  database => "/etc/logstash/GeoLiteCity.dat"
    #  add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
    #  add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
    #}
    #mutate {
    #  convert => [ "[geoip][coordinates]", "float"]
    #}

  }
}

#########
# Audit #
#########
filter {
  if [type] == "audit" {
    grok {
      break_on_match => false
      keep_empty_captures => false

      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/action\":\"%{DATA:action}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/sub-actions\":\"%{DATA:sub_actions}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/properties/add\":\"%{DATA:properties_added}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/properties/delete\":\"%{DATA:properties_deleted}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/properties/from\":\"%{DATA:properties_changed_from}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/properties/to\":\"%{DATA:properties_changed_to}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/aspects/add\":\"%{DATA:aspects_added}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/aspects/delete\":\"%{DATA:aspects_deleted}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/path\":\"%{DATA:node_path}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/type\":\"%{DATA:node_type}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/alfresco-access/transaction/user\":\"%{DATA:transaction_user}\"%{GREEDYDATA}" ]
      match => [ "message", "{\"id\":%{NUMBER:id},\"application\":\"%{GREEDYDATA:application}\",\"user\":\"%{GREEDYDATA:user}\",\"time\":\"%{TIMESTAMP_ISO8601:audit_time}\",\"values\":{\"%{GREEDYDATA:values}%{GREEDYDATA}}"]
    }

    date {
       match => ["audit_time" , "yyyy-MM-dd HH:mm:ss,SSSZZ", "ISO8601"]
       target => "@timestamp"
    }
  }
}

############
# Audit RM #
############
filter {
  if [type] == "auditRM" {
    grok {
      break_on_match => false
      keep_empty_captures => false

      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/identifier\":\"%{DATA:identifier}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/changes/after/value\":\"%{DATA:changesAfter}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/changes/before/value\":\"%{DATA:changesBefore}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/name/value\":\"%{DATA:eventName}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/name\":\"%{DATA:nodeName}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/nodeRef\":\"%{DATA:nodeRef}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/person/fullName\":\"%{DATA:eventPerson}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/person/roles\":\"%{DATA:personRoles}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/type\":\"%{DATA:nodeType}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/namePath\":\"%{DATA:nodePath}\",%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"/RM/event/node/nameRefPath\":\"%{DATA:nodeRefPath}\",%{GREEDYDATA}" ]
      match => [ "message", "{\"id\":%{NUMBER:id},\"application\":\"%{GREEDYDATA:application}\",\"user\":\"%{GREEDYDATA:user}\",\"time\":\"%{TIMESTAMP_ISO8601:audit_time}\",\"values\":{\"%{GREEDYDATA:values}%{GREEDYDATA}}"]
    }

    date {
       match => ["audit_time" , "yyyy-MM-dd HH:mm:ss,SSSZZ", "ISO8601"]
       target => "@timestamp"
    }
  }
}

##############
# Activities #
##############
filter {
  if [type] == "activities" {
    grok {
      break_on_match => false
      keep_empty_captures => false

      match => [ "message", "ID:%{NUMBER:id} %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} POST_DATE:%{TIMESTAMP_ISO8601:date} %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} ACTIVITY_SUMMARY:%{DATA:activitySummary} %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} FEED_USER_ID:%{DATA:userId} %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"firstName\":\"%{DATA:firstName}\"%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"lastName\":\"%{DATA:lastName}\"%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"parentNodeRef\":\"%{DATA:parentNodeRef}\"%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"nodeRef\":\"%{DATA:nodeRef}\"%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"page\":\"%{DATA:page}\"%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA}\"title\":\"%{DATA:title}\"%{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} ACTIVITY_TYPE:%{GREEDYDATA}\.%{DATA:activityType} %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} SITE_NETWORK:%{DATA:siteNetwork} %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} APP_TOOL:%{WORD:appTool}" ]

      match => [ "message", "\[ID:%{NUMBER:id}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[POST_DATE:%{TIMESTAMP_ISO8601:date}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[ACTIVITY_SUMMARY:%{DATA:activitySummary}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[FEED_USER_ID:%{DATA:userId}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[ACTIVITY_TYPE:%{GREEDYDATA}\.%{DATA:activityType}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[SITE_NETWORK:%{DATA:siteNetwork}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[APP_TOOL:%{WORD:appTool}\]" ]
    }

    date {
       match => ["date" , "yyyy-MM-dd HH:mm:ss,SSSZZ", "ISO8601"]
       target => "@timestamp"
    }
  }
}

#############
# Workflows #
#############
filter {
  if [type] == "workflows" {
    grok {
      break_on_match => false
      keep_empty_captures => false

      match => [ "message", "\[ID:%{NUMBER:id}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[TASK_DEF:%{DATA:taskDef}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[PROC_INST_ID:%{DATA:procInstId}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[PARENT_TASK_ID:%{DATA:parentTaskId}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[PROC_DEF_ID:%{DATA:procDefId}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[DESCRIPTION:%{DATA:description}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[NAME:%{DATA:name}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[OWNER:%{DATA:owner}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[ASSIGNEE:%{DATA:assignee}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[START_TIME:%{TIMESTAMP_ISO8601:startTime}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[CLAIM_TIME:%{TIMESTAMP_ISO8601:claimTime}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[END_TIME:%{TIMESTAMP_ISO8601:endTime}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[DUE_DATE:%{TIMESTAMP_ISO8601:dueDate}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[DURATION:%{NUMBER:duration}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[DELETE_REASON:%{DATA:deleteReason}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[PRIORITY:%{DATA:priority}\] %{GREEDYDATA}" ]
      match => [ "message", "%{GREEDYDATA} \[CATEGORY:%{DATA:category}\] %{GREEDYDATA}" ]
    }

    mutate {
      convert => [ "duration", "float" ]
    }

    date {
       match => ["startTime" , "yyyy-MM-dd HH:mm:ss,SSSZZ", "ISO8601"]
       target => "@timestamp"
    }
  }
}

#########
# p6spy #
#########

filter {
  if [type] == "p6spy" {
    grok {
      match => [ "message", "%{DATA:logdate}\|%{NUMBER:executionTime}\|%{DATA:category}\|%{DATA:connectionID}\|%{DATA:statementSQL}\|%{DATA:effectiveSQL}$" ]
    }

    mutate {
      convert => [ "executionTime", "float" ]
    }

    date {
      match => [ "logdate" , "dd-MM-yy:HH:mm:ss:SSS" ]
      target => "@timestamp"
    }

    mutate {
      remove_field => [ 'statementSQL' ]
    }
  }
}

################################
# Filter for Tomcat Access log #
################################

filter {
  if [type] == "TomcatAccessLog" {
    grok {
      match => [ "message", "%{IPORHOST:clientip} %{USER:ident} %{DATA:auth} \[%{HTTPDATE:timestamp}\] \"(%{WORD:verb} %{NOTSPACE:request} (HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent} %{NUMBER:responseTime} \"%{DATA:thread}\"" ]
    }

    mutate {
      convert => [ "responseTime", "float" ]
    }

    date {
      match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
      target => "@timestamp"
    }
  }
}

################
# jstatbeat    #
################
filter {
  if [type] in ["jstatbeat","java8"] {
    json {
      source => "message"
    }

    mutate {
      replace => [ "type", "jstatbeat" ]
    }
  }
}

##############
# packetbeat #
##############
filter {
  if "packetbeat" in [tags] {
    # Remove leading and trailing whitspaces (including newline etc. etc.)
    mutate {
      strip => "method"
    }
  }
}

################
# dstat filter #
################
filter {
  if[type]=="dstatPerformance"{
    grok {
      match => [ "message", "(?<dstat_timestamp>%{MONTHDAY}-%{MONTHNUM} %{HOUR}:%{MINUTE}:%{SECOND}),%{NUMBER:cpu_usr:float},%{NUMBER:cpu_sys:float},%{NUMBER:cpu_idle:float},%{NUMBER:wai:float},%{NUMBER:hiq:float},%{NUMBER:siq:float},%{NUMBER:disk_read:float},%{NUMBER:disk_write:float},%{NUMBER:recv:float},%{NUMBER:send:float},%{NUMBER:paging_in:float},%{NUMBER:paging_out:float},%{NUMBER:sytsem_int:float},%{NUMBER:system_csw:float},%{NUMBER:mem_used:double},%{NUMBER:mem_buff:double},%{NUMBER:mem_cache:double},%{NUMBER:mem_free:double}" ]
    }
  }

  date {
    match => [ "dstat_timestamp", "dd-MM HH:mm:ss", "YYYY-MM-dd HH:mm:ss", "ISO8601" ]
    target => "@timestamp"
  }

  mutate {
    convert => [ "mem_used", "float" ]
    convert => [ "mem_buff", "float" ]
    convert => [ "mem_cache", "float" ]
    convert => [ "mem_free", "float" ]
  }
}

###################
# DB-Health-Check #
###################
filter {
  if[type]=="DB-Health-Check"{

    # Drop empty lines
    if [message] =~ /^[\s]*$/
    {
      drop {}
    }

    grok {
      break_on_match => true
      keep_empty_captures => false
      # Doc per mimetype
      match => [ "message", "(?<DOCS_PER_MIMETYPE>DOCS_PER_MIMETYPE)\|%{DATA:mimeType}\|%{NUMBER:nodes}\|%{NUMBER:size}\|" ]
      # Docs per content store
      match => [ "message", "(?<DOCS_PER_STORE>DOCS_PER_STORE)\|%{NUMBER:nodes}\|%{DATA:date}\|%{DATA:protocol}\|%{DATA:identifier}\|" ]
      # Docs per content type
      match => [ "message", "(?<DOCS_PER_CONTENT_TYPE>DOCS_PER_CONTENT_TYPE)\|%{DATA:contentType}\|%{NUMBER:nodes}\|" ]
      # Large folders
      match => [ "message", "(?<LARGE_FOLDER>LARGE_FOLDER)\|%{NUMBER:nodes}\|%{DATA:nodeRef}\|" ]
      # Process instances
      match => [ "message", "(?<PROCESS_INSTANCES>PROCESS_INSTANCES)\|%{DATA:id}\|%{NUMBER:processes}\|%{DATA:user}\|%{DATA:start_date}\|%{DATA:end_date}\|" ]
      # Task instances
      match => [ "message", "(?<TASK_INSTANCES>TASK_INSTANCES)\|%{NUMBER:tasks}\|%{DATA:id}\|%{DATA:user}\|%{DATA:start_date}\|%{DATA:end_date}\|" ]
      # Individual Site Feed Notification
      match => [ "message", "(?<INDIVIDUAL_FEEDS>INDIVIDUAL_FEEDS)\|%{NUMBER:nodes}\|%{DATA:date}\|%{DATA:site}\|%{DATA:event}\|" ]
      # Group Site Feed Notifications
      match => [ "message", "(?<GROUP_FEEDS>GROUP_FEEDS)\|%{NUMBER:feeds}\|%{DATA:date}\|%{DATA:site}\|%{DATA:event}\|%{NUMBER:nodes}\|" ]
      # Folders with large number of files
      match => [ "message", "(?<LARGE_FOLDER>LARGE_FOLDER)\|%{NUMBER:nodes}\|%{DATA:nodeRef}\|" ]
      # Transactions with large number of nodes
      match => [ "message", "(?<NODES_PER_TRANSACTION>NODES_PER_TRANSACTION)\|%{NUMBER:nodes}\|%{DATA:transactionId}\|" ]
    }
  }

  if [DOCS_PER_STORE] {
    mutate {
      add_field => {
        "store" => "%{protocol}/%{identifier}"
      }
    }
  }

  mutate {
    convert => [ "nodes", "float" ]
    convert => [ "instances", "float" ]
    convert => [ "size", "float" ]
    convert => [ "feeds", "float" ]
    convert => [ "processes", "float" ]
    convert => [ "tasks", "float" ]
  }

  date {
    match => [ "date", "YYYY-MM-dd", "ISO8601" ]
    target => "@timestamp"
  }
  date {
    match => [ "start_date", "YYYY-MM-dd", "ISO8601" ]
    target => "@timestamp"
  }
  date {
    match => [ "end_date", "YYYY-MM-dd", "ISO8601" ]
    target => "@timestamp"
  }
}

# Drop unwanted data
filter {
  mutate {
    remove_field => [ 'message' ]
    remove_field => [ 'msg' ]
    remove_field => [ 'data' ]
    remove_field => [ 'logdate' ]
    remove_tag => [ '_grokparsefailure' ]
  }
}

###########################
# output to elasticsearch #
###########################

output {
  if [type] in ["logstash-jmx", "AlfrescoCPU", "ServerCPU"] {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "jmx-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "logstash-postgres" { #Uncomment for debugging purposes
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-postgres-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "TomcatAccessLog" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-tomcat-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "ApacheAccessLog" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-apache-access-log-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "dstatPerformance" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-dstat-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "auditRM" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-rm-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "activities" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-activities-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "workflows" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-workflows-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] in ["jstatbeat","java8"] {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      codec => "json"
      index => "jstatbeat-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "metricsets" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "metricbeat-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if "packetbeat" in [tags] {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "packetbeat-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "p6spy" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-p6spy-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "audit" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-audit-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "camel" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-camel-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else if [type] == "DB-Health-Check" {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-db-health-check-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  } else {
    #Uncomment for debugging purposes
    #stdout { codec => rubydebug }
    elasticsearch {
      index => "logstash-alfresco-%{+YYYY.MM.dd}"
      hosts => "127.0.0.1:9200"
    }
  }
}