Conditional forwarding in logstash.conf not working

Elastic Stack Logstash
1

Hi People,

I try to get some conditional splitting in logstash conf working, and missing something maybe...
I use this config:

		input {

tcp {
port => 5001
type => syslog
}
udp {
port => 5001
type => syslog
}
}
filter {
if "Drop:" in [message] { grok { match => { "message" => "(?<\d+>)(?\w+\s\d+\s\d+.\d+.\d+)%{SPACE}%{HOSTNAME}%{SPACE}(?\d(.?)(:))%{SPACE}%{SPACE}%{WORD:Severity}%{SPACE}%{SPACE}%{SPACE}%{SPACE}%{SPACE}(?\w+.\w+.\w+)%{SPACE}%{WORD:Action}(?.)%{SPACE}(?.)(?.)%{WORD:Protocol}(?.)(?<interface_IN>(?<=|)(.?)(?=|))(?.)%{IPV4:src}(?.)(?<src_port>\d+)(?.)%{COMMONMAC:mac}(?.)%{IPV4:dst}(?.)(?<dst_port>\d+)(?.)(?.)(?<interface_OUT>(?<=|)(.?)(?=|))(?.)(?.)(?(?<=|)(.?)(?=|))" }}
mutate { remove_field => [ "_version", "@version", "_score", "_type", "separator1", "separator2", "separator3", "separator4", "separator5", "separator6", "separator7", "separator8", "separator9", "separator10", "separator11", "separator12", "separator13", "separator14", "separator15", "separator16", "separator17", "separator18", "separator19", "message", "misc", "port", "type", "score", "rest" ] }}
else { grok { match => { "message" => { "(?<\d+>)(?\w+\s\d+\s\d+.\d+.\d+)%{SPACE}%{HOSTNAME}%{SPACE}(?\d(.?)(:))%{SPACE}%{SPACE}%{WORD:Severity}%{SPACE}%{SPACE}%{SPACE}%{SPACE}%{SPACE}(?\w+.\w+.\w+)%{SPACE}%{WORD:Action}(?.)%{SPACE}%{WORD:Org}(?.)%{WORD:Protocol}(?.)(?<interface_IN>(?<=|)(.?)(?=|))(?.)%{IPV4:src}(?.)(?<src_port>\d+)(?.)%{COMMONMAC:mac}(?.)%{IPV4:dst}(?.)(?<dst_port>\d+)(?.)(?\w+)(?.)(?<interface_OUT>(?<=|)(.?)(?=|))(?.)(?(?<=|)(.?)(?=|))(?.)(?(?<=|)(.?)(?=|))(?.)%{IPV4:src_NAT}(?.)%{IPV4:dst_NAT}(?.)(?(?<=|)(.?)(?=|))(?.)(?<packet_count>(?<=|)(.?)(?=|))(?.)(?(?<=|)(.?)(?=|))(?.)(?(?<=|)(.?)(?=|))(?.)(?(?<=|)(.?)(?=|))(?.)(?(?<=|)(.*?)(?=|))%{GREEDYDATA:rest}" }}}
mutate { remove_field => [ "_version", "@version", "_score", "_type", "separator1", "separator2", "separator3", "separator4", "separator5", "separator6", "separator7", "separator8", "separator9", "separator10", "separator11", "separator12", "separator13", "separator14", "separator15", "separator16", "separator17", "separator18", "separator19", "message", "misc", "port", "type", "score", "rest" ] }}
}
output {
elasticsearch { hosts => ["localhost:9200"] }
file {
path => "C:\Users\Administrator\Desktop\LogTest\test.log"
}
}

and I get this error:
Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, => at line 14, column 1080 (byte 2275) after filter { \n\t\tif "Drop:" in [message]

I seem to miss something but cannot see what exactly. If I comment out the else statement and build the filter with only filter { grok { match => { GROKFILTER}}} than it works... also the if part is working by itself, but the IF THEN I am not able to link together...

thanks for any input in advance!

Regards,
Gergö

2

Can you take the colon out of the "drop:" ? so it is just:

if "Drop" in [message] {
   grok { }
}
3

Thanks for the tip, I just did and get the same error only the colon is not in the error message:

Failed to execute action {:action=>LogStash::PipelineAction::Create/pipeline_id:main, :exception=>"LogStash::ConfigurationError", :message=>"Expected one of #, => at line 15, column 1 (byte 2270) after filter {\n\t\tif "Drop" in [message] { grok { match =>

I think it migh be something with the separation of the if then syntax as it expects something in line 15 column1 in the conf file. that position is a "}" closing in the last brace in the line:
else { grok { match => { "message" => {

4

You have provided a different error message this time, it is now further along the chain than last time. See this time it is line 15 not 14 and the error message is longer.

Could you post your config with proper formatting please? It is very hard to read at the moment! Thanks

5

First of all thanks for hepling me, and sorry I did not get the formatting right...
I think this should do:

 input {
tcp {
port => 5001
type => syslog
}
udp {
port => 5001
type => syslog
} }
filter {
	if "Drop:" in [message] { grok { match => { "message" => "(?<aid><\d+>)(?<Date>\w+\s\d+\s\d+.\d+.\d+)%{SPACE}%{HOSTNAME}%{SPACE}(?<Log>\d(.*?)(\:))%{SPACE}%{SPACE}%{WORD:Severity}%{SPACE}%{SPACE}%{SPACE}%{SPACE}%{SPACE}(?<logsource>\w+.\w+.\w+)%{SPACE}%{WORD:Action}(?<misc>.)%{SPACE}(?<separator0>.)(?<separator1>.)%{WORD:Protocol}(?<separator2>.)(?<interface_IN>(?<=\|)(.*?)(?=\|))(?<separator3>.)%{IPV4:src}(?<separator4>.)(?<src_port>\d+)(?<separator5>.)%{COMMONMAC:mac}(?<separator6>.)%{IPV4:dst}(?<separator7>.)(?<dst_port>\d+)(?<separator9>.)(?<separator10>.)(?<interface_OUT>(?<=\|)(.*?)(?=\|))(?<separator11>.)(?<separator12>.)(?<Info>(?<=\|)(.*?)(?=\|))" }}
mutate { remove_field => [ "_version", "@version", "_score", "_type", "separator1", "separator2", "separator3", "separator4", "separator5", "separator6", "separator7", "separator8", "separator9", "separator10", "separator11", "separator12", "separator13", "separator14", "separator15", "separator16", "separator17", "separator18", "separator19", "message", "misc", "port", "type", "score", "rest" ] }}
else { grok { match => { "message" => { "(?<aid><\d+>)(?<Date>\w+\s\d+\s\d+.\d+.\d+)%{SPACE}%{HOSTNAME}%{SPACE}(?<Log>\d(.*?)(\:))%{SPACE}%{SPACE}%{WORD:Severity}%{SPACE}%{SPACE}%{SPACE}%{SPACE}%{SPACE}(?<logsource>\w+.\w+.\w+)%{SPACE}%{WORD:Action}(?<misc>.)%{SPACE}%{WORD:Org}(?<separator1>.)%{WORD:Protocol}(?<separator2>.)(?<interface_IN>(?<=\|)(.*?)(?=\|))(?<separator3>.)%{IPV4:src}(?<separator4>.)(?<src_port>\d+)(?<separator5>.)%{COMMONMAC:mac}(?<separator6>.)%{IPV4:dst}(?<separator7>.)(?<dst_port>\d+)(?<separator8>.)(?<protocol>\w+)(?<separator9>.)(?<interface_OUT>(?<=\|)(.*?)(?=\|))(?<separator10>.)(?<Rule>(?<=\|)(.*?)(?=\|))(?<separator11>.)(?<Info>(?<=\|)(.*?)(?=\|))(?<separator12>.)%{IPV4:src_NAT}(?<separator13>.)%{IPV4:dst_NAT}(?<separator14>.)(?<duration>(?<=\|)(.*?)(?=\|))(?<separator15>.)(?<packet_count>(?<=\|)(.*?)(?=\|))(?<separator16>.)(?<receivedBytes>(?<=\|)(.*?)(?=\|))(?<separator17>.)(?<sentBytes>(?<=\|)(.*?)(?=\|))(?<separator18>.)(?<receivedPackets>(?<=\|)(.*?)(?=\|))(?<separator19>.)(?<sentPackets>(?<=\|)(.*?)(?=\|))%{GREEDYDATA:rest}" 
} } }
mutate { remove_field => [ "_version", "@version", "_score", "_type", "separator1", "separator2", "separator3", "separator4", "separator5", "separator6", "separator7", "separator8", "separator9", "separator10", "separator11", "separator12", "separator13", "separator14", "separator15", "separator16", "separator17", "separator18", "separator19", "message", "misc", "port", "type", "score", "rest" ] }
		}}
output	{
elasticsearch { hosts => ["localhost:9200"] }
file {
	path => "C:\Users\Administrator\Desktop\LogTest\test.log"
}
}
6

Your formatting could do with some work, I have tidied it up for you. You had an extra bracket, try this:

 input {
	tcp {
		port => 5001
		type => syslog
	}
	udp {
		port => 5001
		type => syslog
	} 
}
filter {
	if "Drop:" in [message] { 
		grok { 
			match => { "message" => "(?<aid><\d+>)(?<Date>\w+\s\d+\s\d+.\d+.\d+)%{SPACE}%{HOSTNAME}%{SPACE}(?<Log>\d(.*?)(\:))%{SPACE}%{SPACE}%{WORD:Severity}%{SPACE}%{SPACE}%{SPACE}%{SPACE}%{SPACE}(?<logsource>\w+.\w+.\w+)%{SPACE}%{WORD:Action}(?<misc>.)%{SPACE}(?<separator0>.)(?<separator1>.)%{WORD:Protocol}(?<separator2>.)(?<interface_IN>(?<=\|)(.*?)(?=\|))(?<separator3>.)%{IPV4:src}(?<separator4>.)(?<src_port>\d+)(?<separator5>.)%{COMMONMAC:mac}(?<separator6>.)%{IPV4:dst}(?<separator7>.)(?<dst_port>\d+)(?<separator9>.)(?<separator10>.)(?<interface_OUT>(?<=\|)(.*?)(?=\|))(?<separator11>.)(?<separator12>.)(?<Info>(?<=\|)(.*?)(?=\|))" }
		}
		mutate { 
			remove_field => [ "_version", "@version", "_score", "_type", "separator1", "separator2", "separator3", "separator4", "separator5", "separator6", "separator7", "separator8", "separator9", "separator10", "separator11", "separator12", "separator13", "separator14", "separator15", "separator16", "separator17", "separator18", "separator19", "message", "misc", "port", "type", "score", "rest" ] 
		}
	} else { 
		grok { 
			match => { "message" => "(?<aid><\d+>)(?<Date>\w+\s\d+\s\d+.\d+.\d+)%{SPACE}%{HOSTNAME}%{SPACE}(?<Log>\d(.*?)(\:))%{SPACE}%{SPACE}%{WORD:Severity}%{SPACE}%{SPACE}%{SPACE}%{SPACE}%{SPACE}(?<logsource>\w+.\w+.\w+)%{SPACE}%{WORD:Action}(?<misc>.)%{SPACE}%{WORD:Org}(?<separator1>.)%{WORD:Protocol}(?<separator2>.)(?<interface_IN>(?<=\|)(.*?)(?=\|))(?<separator3>.)%{IPV4:src}(?<separator4>.)(?<src_port>\d+)(?<separator5>.)%{COMMONMAC:mac}(?<separator6>.)%{IPV4:dst}(?<separator7>.)(?<dst_port>\d+)(?<separator8>.)(?<protocol>\w+)(?<separator9>.)(?<interface_OUT>(?<=\|)(.*?)(?=\|))(?<separator10>.)(?<Rule>(?<=\|)(.*?)(?=\|))(?<separator11>.)(?<Info>(?<=\|)(.*?)(?=\|))(?<separator12>.)%{IPV4:src_NAT}(?<separator13>.)%{IPV4:dst_NAT}(?<separator14>.)(?<duration>(?<=\|)(.*?)(?=\|))(?<separator15>.)(?<packet_count>(?<=\|)(.*?)(?=\|))(?<separator16>.)(?<receivedBytes>(?<=\|)(.*?)(?=\|))(?<separator17>.)(?<sentBytes>(?<=\|)(.*?)(?=\|))(?<separator18>.)(?<receivedPackets>(?<=\|)(.*?)(?=\|))(?<separator19>.)(?<sentPackets>(?<=\|)(.*?)(?=\|))%{GREEDYDATA:rest}" } 
		} 
	}
	mutate { 
		remove_field => [ "_version", "@version", "_score", "_type", "separator1", "separator2", "separator3", "separator4", "separator5", "separator6", "separator7", "separator8", "separator9", "separator10", "separator11", "separator12", "separator13", "separator14", "separator15", "separator16", "separator17", "separator18", "separator19", "message", "misc", "port", "type", "score", "rest" ] 
	}
	}
output	{
	elasticsearch { hosts => ["localhost:9200"] }
	file {
		path => "C:\Users\Administrator\Desktop\LogTest\test.log"
	}
}
1 Like
7

Dear Lewis,

now its working...! thank you for your help, learned again something! you are awesome!

8

Glad to hear it! Please mark as solved :slight_smile:!

9

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.