Extract value from path in logstash

Elastic Stack Elasticsearch
1

Hi need to extract value from path in logstash,
here is my logpath:
/data/app/20230707/*/*

/data/app1/20230707/host1/*.log
/data/app2/20230707/host2/*.log

need to extract these field from path (FYI: hostname must be overwrite host)
data=constant
app=variable means app name
date of log=20230707
hostname=host1 or host2

here is my logstash config:

input {
  file {
    path => "/data/app/20230707/*/*"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}


filter {
  if [message] =~ /\[SqlExceptionHelper\] SQL (Error|Warning Code):/ {
    grok {
      match => {
        "message" => [
          "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} %{DATA:thread} \[SqlExceptionHelper\] SQL Error: -%{INT:db_errorcode1}, SQLState: %{WORD:sql_state1}",
          "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:loglevel} %{DATA:thread} \[SqlExceptionHelper\] SQL Warning Code: %{INT:db_errorcode2}, SQLState: %{WORD:sql_state2}"
        ]
      }
    }

    date {
      match => ["timestamp", "YYYY-MM-dd HH:mm:ss,SSS"]
    }
  } else if [message] =~ /(AMQ|ARJUNA|COM|EJBCLIENT|ELY|HCANN|HHH|HSEARCH|HV|IJ|ISNPHIB|ISPN|JBERET|JBREM|JBTHR|JBWEB|JBWS|JIPI|JNDIWFHTTP|MODCLUSTER|MSC|PBOX|PROBE|RESTEASY|TXNWFHTTP|UT|UTJS|VFS|WELD|WFCMTOOL|WFHTTP|WFHTTPEJB|WFLY|WFMIGRCLI|WFNAM|WFSM|WFTXN|XNIO|jlibaio)/ {
    grok {
      match => { "message" => ".*\b(?<jboss_errors>(?:AMQ\w*|ARJUNA\w*|COM\w*|EJBCLIENT\w*|ELY\w*|HCANN\w*|HHH\w*|HSEARCH\w*|HV\w*|IJ\w*|ISNPHIB\w*|ISPN\w*|JBERET\w*|JBREM\w*|JBTHR\w*|JBWEB\w*|JBWS\w*|JIPI\w*|JNDIWFHTTP\w*|MODCLUSTER\w*|MSC\w*|PBOX\w*|PROBE\w*|RESTEASY\w*|TXNWFHTTP\w*|UT\w*|UTJS\w*|VFS\w*|WELD\w*|WFCMTOOL\w*|WFHTTP\w*|WFHTTPEJB\w*|WFLY\w*|WFMIGRCLI\w*|WFNAM\w*|WFSM\w*|WFTXN\w*|XNIO\w*|jlibaio\w*)\b).*" }
    }
    grok {
      match => { "message" => "%{TIMESTAMP_ISO8601:timestamp}\s+%{LOGLEVEL:loglevel}\s+%{DATA:id}\s+\[%{DATA:class}\]" }
    }
    if "_grokparsefailure" in [tags] {
      drop { }
    }
    mutate {
      remove_field => ["message", "@version", "event"] # Optionally remove unnecessary fields
    }
    date {
      match => ["timestamp", "YYYY-MM-dd HH:mm:ss,SSS"]
    }
  }
  else {
    drop {}
  }
}

Any idea?
Thanks,

1 Like
2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.