Field have the same value

Elastic Stack Logstash
1

I don't know why logstash keep parsing the same value for the different field.

Here I attach my logstash configuration

input {
file {
start_position => "beginning"
sincedb_path => "/dev/null"
path => [ "/home/elasticsearch/logstash-7.10.0/data/tomcat.log"]
type => "tomcat"
codec => multiline {
      pattern => "^\s"
      what => "next"
}

}
}

filter {
if [type] == "tomcat" {
grok {
break_on_match => false
  match => [
    "message", "%{URIPATH:request}",
    "message", "%{NOTSPACE:available_pattern}",
    "message", "%{QS:agent}",
    "message", "%{WORD:method}",
    "message", "%{NOTSPACE:type}",
    "message", "%{NOTSPACE:referrer}",
    "message", "%{NOTSPACE:source-type}",
    "message", "%{NUMBER:size}",
    "message", "%{NUMBER:responseTime}",
    "message", "%{NOTSPACE:host}",
    "message", "%{IP:client}",
    "message", "%{NUMBER:timestamp}\n"
 ]
}

date {
  match => ["timestamp",
    "MMM dd HH:mm:ss",
    "MMM  d HH:mm:ss",
    "MMM dd yyyy HH:mm:ss",
    "MMM  d yyyy HH:mm:ss"
  ]
  timezone => "America/New_York"
}

 if "_grokparsefailure" not in [tags] {
     mutate {
      rename => ["tomcat_message", "message"]
     remove_field => ["timestamp"]
  }
 }
}
}

output {
 elasticsearch {
  index => "tomcat-sample5"
  hosts => ["192.168.1.10:9200"]
 }
}

the result

{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "tomcat-sample5",
"_type" : "_doc",
"_id" : "ZFLTXnYB7OqhOM-iv5cW",
"_score" : 1.0,
"_source" : {
"request" : "/mapi/sell/registrasi_mdn_new",
"referrer" : ""request":",
"size" : "2.1",
"responseTime" : "2.1",
"message" : """ "request": "/mapi/sell/registrasi_mdn_new",
"available_pattern": true,
"agent": "Dalvik/2.1.0 (Linux; U; Android 8.1.0; CPH1803 Build/OPM1.171019.026)",
"method": "POST",
"type": "httpd-access_log",
"referrer": "-",
"source:type": "web-server",
"original_string": "<158>Nov 26 22:59:40 vm-sris01 httpd-access_log 10.14.37.6 - - [26/Nov/2020:22:59:38 +0700] "POST /mapi/sell/registrasi_mdn_new HTTP/1.1" 200 83 "-" "Dalvik/2.1.0 (Linux; U; Android 8.1.0; CPH1803 Build/OPM1.171019.026)" "-" 724800",
"size": 83,
"response": 200,
"host": "vm-sris01",
"ip_src_addr": "10.14.37.6",
"timestamp": 1606406378000""",
"method" : "request",
"@version" : "1",
"source-type" : ""request":",
"available_pattern" : ""request":",
"@timestamp" : "2020-12-14T01:18:19.461Z",
"host" : [
"personal-vm",
""request":"
],
"tags" : [
"multiline",
"_grokparsefailure"
],
"type" : [
"tomcat",
""request":"
],
"path" : "/home/elasticsearch/logstash-7.10.0/data/tomcat.log",
"agent" : ""request"",
"client" : "10.14.37.6"
}
}
]
}
}

Can anyone pls help?

`

2

Your grok filter matches 12 patterns against the [message] field. But all of the patterns are independent. If you try to match the same pattern more than once, as you do with

"message", "%{NOTSPACE:type}",
"message", "%{NOTSPACE:referrer}",
"message", "%{NOTSPACE:source-type}",

then each one will match the same part of the message field, so it will result in the same value.

3

then what should I do?

4

I need them to be different and the value of the field match with the logs

5

A search of the forums should turn up examples of parsing tomcat logs using grok, such as this one.

6

but my logs look like this

{
 "request": "/mapi/sell/registrasi_mdn_new",
 "available_pattern": true,
 "agent": "Dalvik/2.1.0 (Linux; U; Android 8.1.0; CPH1803 Build/OPM1.171019.026)",
"method": "POST",
"type": "httpd-access_log",
 "referrer": "-",
 "source:type": "web-server",
"original_string": "<158>Nov 26 22:59:40 vm-sris01 httpd-access_log 10.14.37.6 - - [26/Nov/2020:22:59:38 +0700] \"POST /mapi/sell/registrasi_mdn_new HTTP/1.1\" 200 83 \"-\" \"Dalvik/2.1.0 (Linux; U; Android 8.1.0; CPH1803 Build/OPM1.171019.026)\" \"-\" 724800",
"size": 83,
"response": 200,
"host": "vm-sris01",
"ip_src_addr": "10.14.37.6",
"timestamp": 1606406378000
}
7

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.