I have setup ELK, where i am trying to parse multiple log files. I am transferring the logs file from my production servers to the server where ELK is installed using RSYNC.
Now, the problem is logstash is parsing one particular log multiple time (16 times). I want this log to be parsed only once.
I am not using filebeat.
I am executing the logstash as "./logstash -f logstash.conf".
Content of logstash conf is as below:
input
{
file
{
path => "/home/finassure/datadog/logs/node1/LISRVR_CDCI_SWIF_*.log"
start_position => "beginning"
sincedb_path => "/home/finassure/ELK/lastlog"
type => "cbclog1"
codec => multiline
{
pattern => "^(Pid:\s[0-9]{3}|[0-9]{4}|[0-9]{5}|[0-9]{6}\sReceived At:)|^(Pid:\s[0-9]{3}|[0-9]{4}|[0-9]{5}|[0-9]{6}\sSent At:)|^(MessageId:)|^(Field\s[0-9][0-9][0-9]:)"
negate => false
what => "next"
}
}
}
filter {
if [type] == "cbclog1" {
if "Received At:" in [message]
{
mutate {
add_field => { "cdci_msg_type" => "request" }
}
}
if "Sent At:" in [message]
{
mutate {
add_field => { "cdci_msg_type" => "response" }
}
}
if [cdci_msg_type] not in ["request", "response"] {
drop { }
}
mutate {
gsub => ['message', "\n", " "]
gsub => ['message', "\t", " "]
}
if [cdci_msg_type] in ["response","request"] {
grok {
match => {
message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-5][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 033)\:\s+(?<cdci_field_033>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+(Field 042)\:\s+(?<cdci_field_042>(\S+))\s+(Field 043)\:\s+(?<cdci_field_043>(\S+))\s+(Field 049)\:\s+(?<cdci_field_049>(\S+))\s+(Field 059)\:\s+(?<cdci_field_059>(\S+))\s+(%{GREEDYDATA:custom})"
}
overwrite => ["message"]
add_field => { "msg_type" => "cbclogs" }
break_on_match => "true"
}
}
if "_grokparsefailure" in [tags]
{
grok
{
match => { message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 033)\:\s+(?<cdci_field_033>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 038)\:\s+(?<cdci_field_038>(\S+))\s+(Field 039)\:\s+(?<cdci_field_039>(\S+))\s+(%{GREEDYDATA:custom })" }
remove_tag => ["_grokparsefailure"]
add_field => { "msg_type" => "cbclogs" }
break_on_match => "true"
}
}
if "_grokparsefailure" in [tags]
{
grok
{
match =>
{
message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 033)\:\s+(?<cdci_field_033>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+(Field 042)\:\s+(?<cdci_field_042>(\S+))\s+(Field 043)\:\s+(?<cdci_field_043>(\S+))\s+(Field 049)\:\s+(?<cdci_field_049>(\S+))\s+"
}
remove_tag => ["_grokparsefailure"]
add_field => { "msg_type" => "cbclogs" }
break_on_match => "true"
}
}
if "_grokparsefailure" in [tags]
{
grok
{
match => { "message" => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+(Field 042)\:\s+(?<cdci_field_042>(\S+))\s+(Field 049)\:\s+(?<cdci_field_049>(\S+))\s+(Field 056)\:\s+(?<cdci_field_056>(\S+))\s+(Field 059)\:\s+(?<cdci_field_059>(\S+))\s+" }
remove_tag => ["_grokparsefailure"]
add_field => { "msg_type" => "cbclogs" }
break_on_match => "true"
}
}
}
}
}
output {
stdout { codec => rubydebug }
if [type] == "cbclog1"
{
elasticsearch {
hosts => "10.0.100.167:9200"
index => "cbclog1"
}
}
}