Elasticsearch message format

Elastic Stack Logstash
1

I need to remove several fields before pushing data to Elasticsearch. I can't use mutate filter because in aggregate filter I'm using field which I need to remove
Conf example:

input {
	jdbc {
		bla_bla_bla
	}
}

filter {
	date {
		match => [ "timestamp", "YYYYMMddHHmmss", "YYYY-MM-dd HH:mm:ss", "ISO8601" ]
		target => "timestamp"
	}
	if [event_name] == "submitted" {
		aggregate {
			task_id => "%{step_id}"
			code => "map['step_duration'] = event.get('timestamp')"
			map_action => "create"
		}
	}
	if [event_name] == "process_end" {
		aggregate {
			add_field => { "step_name" => "step_ololo" }
			task_id => "%{step_id}"
			code => "event.set('step_duration', event.get('timestamp') - map['step_duration'])"
			map_action => "update"
			end_of_task => true
			timeout => 3600
		}
	}
}

output {
	if ([step_name]) {
		elasticsearch {
			ssl_certificate_verification => "false"
			ssl => "false"
			index => "events-%{+YYYY.MM.dd}"
			hosts => [ "localhost:9200" ]
			user => "ololo_user"
			password => "ololo_pass"
		}
	}
}

Need to remove event_name field

2

Can't you just remove the field after the aggregate filter?

3

Doesn't work:

input {
	jdbc {
		bla_bla_bla
	}
}

filter {
	date {
		match => [ "timestamp", "YYYYMMddHHmmss", "YYYY-MM-dd HH:mm:ss", "ISO8601" ]
		target => "timestamp"
	}
	if [event_name] == "submitted" {
		aggregate {
			task_id => "%{step_id}"
			code => "map['step_duration'] = event.get('timestamp')"
			map_action => "create"
		}
	}
	if [event_name] == "process_end" {
		aggregate {
			add_field => { "step_name" => "step_ololo" }
			task_id => "%{step_id}"
			code => "event.set('step_duration', event.get('timestamp') - map['step_duration'])"
			map_action => "update"
			end_of_task => true
			timeout => 3600
		}
	}
        mutate {
            remove_field => [ "%{event_name}" ]
        }
}

output {
	if ([step_name]) {
		elasticsearch {
			ssl_certificate_verification => "false"
			ssl => "false"
			index => "events-%{+YYYY.MM.dd}"
			hosts => [ "localhost:9200" ]
			user => "ololo_user"
			password => "ololo_pass"
		}
	}
}
4 
remove_field => [ "%{event_name}" ]

This will remove the field named according to the contents of the event_name field, i.e. if that field contains "foo" Logstash will attempt to remove the field foo. Do this instead:

 remove_field => [ "event_name" ]
1 Like
5

Thanks a lot. Works now!

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.