Help! logstash conf filter add_tag not working

Hey all.
This is my first post in these forums, so- hey everyone :slight_smile:
I am parsing few different input file sources using grok filters.
if there is no match on any of them I have created a "fail safe" while adding a tag "no_match".
if filter "ccc" have had a hit I want to add tag "ccc".

I have managed to see matches both from the "failsafe" and the "ccc" file, but no tag were added.
I do see "multiline" tag some times, but none of my manual tags are being added.
Please see my part of my Logstash configuration below, I have removed some data for privacy.
Thank you.
Edit: I am using Logstash and ES version 6.1.1

input {
file {
	path => "aaa.log*"
	start_position => "beginning"
    codec => multiline {
		pattern => "^%{TIMESTAMP_ISO8601}"
		negate => true
		what => "previous"
		max_lines => 1000
	}
	type => "aaa"
}
file {
	path => "bbb.log*"
	start_position => "beginning"
    codec => multiline {
		pattern => "^%{TIMESTAMP_ISO8601}"
		negate => true
		what => "previous"
		max_lines => 1000
	}
	type => "bbb"
}
file {
	path => "ccc.log*"
	start_position => "beginning"
    codec => multiline {
		pattern => "^%{TIMESTAMP_ISO8601}"
		negate => true
		what => "previous"
		max_lines => 1000
	}
	type => "ccc"
}
}
filter {
if [type] == "aaa" {
	grok {
		break_on_match => true
		#With exception stacktrace:
		match => {
			"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}\[%{DATA:correlation_id}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:tenant}\]%{SPACE}\[%{DATA:organization}\]%{SPACE}\[%{DATA:user}\]%{SPACE}-((%{SPACE}<)%{GREEDYDATA:text}>(\n|\r\n)(?m)%{GREEDYDATA:exception})"
			"path" => "%{GREEDYDATA:filename}"
		}
		#Without stacktrace:
		match => {
				"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}\[%{DATA:correlation_id}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:tenant}\]%{SPACE}\[%{DATA:organization}\]%{SPACE}\[%{DATA:user}\]%{SPACE}-%{SPACE}<%=@lt%>{GREEDYDATA:text}>"
				"path" => "%{GREEDYDATA:filename}"
		}
		#failsafe
		match => {
				"message" => "%{GREEDYDATA:text}"
				"path" => "%{GREEDYDATA:filename}"
				add_tag => ["no_match"]
		}
	}
}
if [type] == "bbb" {
	grok {
		break_on_match => true
		#With exception stacktrace:
		match => {
			"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}-((%{SPACE}<)%{GREEDYDATA:text}>(\n|\r\n)(?m)%{GREEDYDATA:exception})"
			"path" => "%{GREEDYDATA:filename}"
		}
		#Without stacktrace:
		match => {
				"message" => "%{TIMESTAMP_ISO8601:event_timestamp}%{SPACE}%{DATA:log_level}%{SPACE}\[%{DATA:thread_name}\]%{SPACE}\[%{DATA:host}\]%{SPACE}\[%{DATA:session}\]%{SPACE}\[%{DATA:trx}\]%{SPACE}\[%{DATA:java_class}\]%{SPACE}-%{SPACE}<%=@lt%>{GREEDYDATA:text}>"
				"path" => "%{GREEDYDATA:filename}"
		}
		#failsafe
		match => {
				"message" => "%{GREEDYDATA:text}"
				"path" => "%{GREEDYDATA:filename}"
				add_tag => ["no_match"]
		}
	}
}
if [type] == "ccc" {
	grok {
		break_on_match => true
		match => {
				"message" => "%{TIMESTAMP_ISO8601:event_timestamp},%{DATA:tenant},%{DATA:organization},%{DATA:request},%{DATA:correlation_id},%{NUMBER:time:int}(,,|,|)(%{SPACE}\[%{GREEDYDATA:text}\]|)"
				"path" => "%{GREEDYDATA:filename}"
				add_tag => ["ccc"]
		}
		#failsafe
		match => {
				"message" => "%{GREEDYDATA:text}"
				"path" => "%{GREEDYDATA:filename}"
				add_tag => ["no_match"]
		}
	}
}
date {
	match => [ "event_timestamp", "YYYY-MM-dd HH:mm:ss.SSS" ]
	timezone => "Etc/GMT"
	locale => "en"
}
output { ElasticSearch info}

Having multiple occurrences of an option often comes with surprising results. Also, your add_tag is inside the set of matches, so it is trying to match the field [add_tag] to the pattern "no_match".

Hey Badger,
Thank you for writing back!

So, how do you suggest to mark or tag log lines from a file that didn't match the pattern?
I don't want to lose them. i want to learn what didn't match and then to write a new match for those lines.

Is there a better way to parse a log file?

By default a _grokparsefailure tag is added if none of the patterns match.

Okay,
Thank you.

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.