The following config can run smoothly
filter {
if(" - response:" in [message]){
grok {
match => { "message" => "%{GREEDYDATA:timestamp} %{GREEDYDATA:thread} INFO - response:%{GREEDYDATA:serviceName} %{GREEDYDATA:serviceMethod} %{GREEDYDATA:sessionId} %{GREEDYDATA:ipAddress} %{GREEDYDATA:callerFrom} %{GREEDYDATA:operatorId} %{GREEDYDATA:operat orName} %{GREEDYDATA:customerId} %{GREEDYDATA:customerName} %{NUMBER:timeConsumed}ms"}
}
date {
match => [ "timestamp", "MM-dd HH:mm:ss SSS" ]
}
mutate { remove_field => "thread" }
mutate { add_field => { log_type => "ksresponse" } }
} else if(" - SQL Preparing: " in [message]){
grok {
match => { "message" => "%{GREEDYDATA:timestamp} %{GREEDYDATA:thread} DEBUG - SQL Preparing: %{GREEDYDATA:sql} args:%{GREEDYDATA:sqlArgs}"}
}
date {
match => [ "timestamp", "MM-dd HH:mm:ss SSS" ]
}
mutate { remove_field => "thread" }
mutate { add_field => { log_type => "sql" } }
}
else{
drop{}
}
}
While with this one , the logstash can just collect the data before 1 am
filter {
if(" - request:" in [message]){
grok {
match => { "message" => "%{GREEDYDATA:timestamp} %{GREEDYDATA:thread} INFO - request:%{GREEDYDATA:serviceName} %{GREEDYDATA:serviceMethod} %{GREEDYDATA:sessionId} %{GREEDYDATA:ipAddress} %{GREEDYDATA:callerFrom} %{GREEDYDATA:operatorId} %{GREEDYDATA:operato rName} %{GREEDYDATA:customerId} %{GREEDYDATA:customerName} reqArgs:%{GREEDYDATA:reqArgs}"}
}
date {
match => [ "timestamp", "MM-dd HH:mm:ss SSS" ]
}
mutate { remove_field => "thread" }
mutate { add_field => { log_type => "ksrequest" } }
}else
if(" - response:" in [message]){
grok {
match => { "message" => "%{GREEDYDATA:timestamp} %{GREEDYDATA:thread} INFO - response:%{GREEDYDATA:serviceName} %{GREEDYDATA:serviceMethod} %{GREEDYDATA:sessionId} %{GREEDYDATA:ipAddress} %{GREEDYDATA:callerFrom} %{GREEDYDATA:operatorId} %{GREEDYDATA:operat orName} %{GREEDYDATA:customerId} %{GREEDYDATA:customerName} %{NUMBER:timeConsumed}ms"}
}
date {
match => [ "timestamp", "MM-dd HH:mm:ss SSS" ]
}
mutate { remove_field => "thread" }
mutate { add_field => { log_type => "ksresponse" } }
}else if(" - SQL Preparing: " in [message]){
grok {
match => { "message" => "%{GREEDYDATA:timestamp} %{GREEDYDATA:thread} DEBUG - SQL Preparing: %{GREEDYDATA:sql} args:%{GREEDYDATA:sqlArgs}"}
}
date {
match => [ "timestamp", "MM-dd HH:mm:ss SSS" ]
}
mutate { remove_field => "thread" }
mutate { add_field => { log_type => "sql" } }
}
else{
drop{}
}
}
