Hi, I am using Logstash 5.4.1. I don't why kv include_keys doesn't work as expected. could you please provide some advice? if there is some error in my config or this is known issue? Many thanks!
below is my config:
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:syslog_time}%{SPACE}[%{DATA:thread}]%{SPACE}%{LOGLEVEL:level}%{SPACE}[%{GREEDYDATA:kvpairs}]%{SPACE}%{JAVACLASS:class}%{SPACE}-%{SPACE}%{JAVALOGMESSAGE:java_message}" }
add_field => { "msg_subType" => "java" }
}
kv {
source => "kvpairs"
trim_key => "<>[],:"
include_keys => ["requestId", "endPoint", "accountNumber", "customerId"]
remove_field => ["kvpairs"]
field_split => ","
}
this is logstash output, as you can see "kvpairs" not removed.
{
"kvpairs" => "endpoint=GET /goals, sourceIp=10.195.182.201, requestId=30b12b66-dc78-4c94-a8a6-2ce0c1784a92, customerId=3103, cpf=48814300178, accountNumber=6542, branch=3750, deviceId=6O2SVU/aIu6iiityPtPg+UvOu7+wyorRYWuqNqiecn9KD6QZeJ7EaGVbNOmijzGl, jti=001D46440886BF5X0260F953C3A43F55",
"syslog_time" => "2017-05-21 11:26:08",
"offset" => 462,
"level" => "DEBUG",
"input_type" => "log",
"msg_subType" => "java",
"source" => "/opt/sit_log/small_2.log",
"thread" => "http-/0.0.0.0:8080-10",
"message" => "2017-05-21 11:26:08 [http-/0.0.0.0:8080-10] DEBUG [endpoint=GET /goals, sourceIp=10.195.182.201, requestId=30b12b66-dc78-4c94-a8a6-2ce0c1784a92, customerId=3103, accountNumber=6542, branch=3750, deviceId=6O2SVU/aIu6iiityPtPg+UvOu7+wyorRYWuqNqiecn9KD6QZeJ7EaGVbNOmijzGl, jti=001D46440886BF5X0260F953C3A43F55] b.c.b.n.p.c.dao.LoggableJdbcTemplate - Executing prepared SQL statement [SELECT * FROM GOAL_TYPE e WHERE e.GOAL_TYPE_UID = ?]",
"type" => "syslog",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"@timestamp" => 2017-06-14T18:02:26.934Z,
"java_message" => "Executing prepared SQL statement [SELECT * FROM GOAL_TYPE e WHERE e.GOAL_TYPE_UID = ?]",
"@version" => "1",
"beat" => {
"hostname" => "ip-172-31-0-204",
"name" => "ip-172-31-0-204",
"version" => "5.4.1"
},
"host" => "ip-172-31-0-204",
"class" => "b.c.b.n.p.c.dao.LoggableJdbcTemplate"
}
if I changed above config, just remove line "include_keys", it works fine, all keys in kvpairs generated properly.
config:
kv {
source => "kvpairs"
trim_key => "<>[],:"
remove_field => ["kvpairs"]
field_split => ","
}
below is logstash output:
{
"syslog_time" => "2017-05-21 11:26:08",
" accountNumber" => "6602",
"msg_subType" => "java",
"source" => "/opt/sit_log/small_3.log",
"type" => "syslog",
"endpoint" => "GET /lime/termsAndConditions",
"java_message" => "Executing prepared SQL statement [SELECT * FROM PRODUCT p WHERE p.PRODUCT_CD = ?]",
"@version" => "1",
"beat" => {
"hostname" => "ip-172-31-0-204",
"name" => "ip-172-31-0-204",
"version" => "5.4.1"
},
"host" => "ip-172-31-0-204",
" deviceId" => "YUx9wEhw0kdCbTuT0dCrgbUC5pyKZWG8b62C84fp16zAhrCo0oHj+lL8Hqr7AOzS",
"class" => "b.c.b.n.p.c.dao.LoggableJdbcTemplate",
" requestId" => "d4bf121e-b957-4b0a-8bc8-4327f3648fae",
"offset" => 915,
"level" => "DEBUG",
" sourceIp" => "10.195.182.201",
"input_type" => "log",
"thread" => "http-/0.0.0.0:8080-12",
"message" => "2017-05-21 11:26:08 [http-/0.0.0.0:8080-12] DEBUG [endpoint=GET /lime/termsAndConditions, sourceIp=10.195.182.201, requestId=d4bf121e-b957-4b0a-8bc8-4327f3648fae, customerId=3413, accountNumber=6602, branch=3750, deviceId=YUx9wEhw0kdCbTuT0dCrgbUC5pyKZWG8b62C84fp16zAhrCo0oHj+lL8Hqr7AOzS, jti=001D46440886BF5X0260F953C3A43F54] b.c.b.n.p.c.dao.LoggableJdbcTemplate - Executing prepared SQL statement [SELECT * FROM PRODUCT p WHERE p.PRODUCT_CD = ?]",
" customerId" => "3413",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
" jti" => "001D46440886BF5X0260F953C3A43F54",
"@timestamp" => 2017-06-14T18:10:26.948Z,
" branch" => "3750"
}