Output merged Logfile data to MongoDB using Logstash

Hi everyone,

I am quite new to Logstash and have been learning to use it for some time. I have a log file as follows

logfile.txt
01012014:123456789:1:INFO:Some Text Type1:TranLine
01012014:123456790:1:INFO:Some Text Type2:TranLine
01012014:123456790:1:INFO:StartofLogging:TranStart
01012014:123456791:1:INFO:Log text Type 1:TranLine
01012014:123456792:1:INFO:Log text Type 2:TranLine
01012014:123456793:1:ERROR:Log text Type 3:TranLine
01012014:123456800:1:INFO:EndofLogging:TranEnd
01012014:123456810:1:INFO:StartofLogging:TranStart
01012014:123456810:1:INFO:Log text Type 1:TranLine
01012014:123456811:1:INFO:Log text Type 21:TranLine
01012014:123456812:1:INFO:EndofLogging:TranEnd

I want to be able to include this into a mongoDB database in 3 new lines
01012014:123456789:1:INFO:Some Text Type1, 01012014:123456790:1:INFO:Some Text Type2
01012014:123456790:1:INFO:StartofLogging, 01012014:123456791:1:INFO:Log text Type 1,01012014:123456792:1:INFO:Log text Type 2, 01012014:123456793:1:ERROR:Log text Type 3,01012014:123456800:1:INFO:EndofLogging:TranEnd
01012014:123456810:1:INFO:StartofLogging, 01012014:123456810:1:INFO:Log text Type 1, 01012014:123456811:1:INFO:Log text Type 21, 01012014:123456812:1:INFO:EndofLogging

However I am unable to get the desired output. What I get is as follows
[ "01012014:123456789:1:INFO:Some Text Type1,"]
[ "01012014:123456790:1:INFO:Some Text Type2,"]
[ "16092016:123456790:1:INFO:StartofLogging,"]
[ "01012014:123456791:1:INFO:Log Text Type 1,"]
[ "01012014:123456792:1:INFO:Log Text Type 2,"]
[ "01012014:123456793:1:ERROR:Log Text Type 3,"]
[ "01012014:123456800:1:INFO:EndofLogging,"]
[ "16092016:123456810:1:INFO:StartofLogging,"]
[ "01012014:123456810:1:INFO:Log Text Type 1,"]
[ "01012014:123456811:1:INFO:Log Text Type 21,"]
[ "01012014:123456812:1:INFO:EndofLogging,"]

The configurations I have used are as follows
patterns
TASKID %{NUMBER}
TXTLINE %{NUMBER}:%{NUMBER}:%{LOGLEVEL}:%{GREEDYDATA}

config file
input {
file {
path => "D:/logstash-6.0.0/data/logfile.txt"
start_position => "beginning"
sincedb_path => "/dev/null"
}
}

filter {
grok {
patterns_dir => "./patterns"
match => [ "message", "%{TASKID:taskID}:%{TXTLINE:text}:%{WORD:logger}" ]
}
mutate {
add_field => { "transaction" => "%{taskID}:%{text}," }
}

if [logger] == "TranLine" {
	mutate {
		merge => { "transaction" => "%{taskID}:%{text}," }
		remove_field => [ "taskID" ]
		remove_field => [ "text" ]
		remove_field => [ "message" ]
		remove_field => [ "logger" ]
	}
}
else {
	if [logger] == "TranStart" {
		mutate {
			merge => { "transaction" => "%{taskID}:%{text}," }
			remove_field => [ "taskID" ]
			remove_field => [ "text" ]
			remove_field => [ "message" ]
			remove_field => [ "logger" ]
		}	
	}
	else {
		if [logger] == "TranEnd" {
			mutate {
				merge => { "transaction" => "%{taskID}:%{text}," }
				remove_field => [ "taskID" ]
				remove_field => [ "text" ]
				remove_field => [ "message" ]
				remove_field => [ "logger" ]
			}	
		}
	}
}

}

output {
mongodb {
collection => "Logfiles"
database => "test"
uri => "mongodb://localhost:27017/"
}
}

Can someone please guide me.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.