How to get filter value metadata in input

Elastic Stack Logstash
1

Hi Guys,

input{
kafka {
bootstrap_servers => "172.20.188.11:9092,172.20.188.31:9092,172.20.188.30:9092"
topics => ["DS_Test"]
decorate_events => true
group_id => "ds_test"
codec => json
max_poll_records => "100"
consumer_threads => 2
session_timeout_ms => "30000"
auto_offset_reset => "earliest"
client_id => "logstash-ingestion"
}
}

filter {
mutate { remove_field => "@Version" }
mutate { remove_field => "@timestamp" }
mutate { add_field => { "[@metadata][index]" => "%{[eventHeader][bpcName]}%{[eventHeader][applicationId]}"} }
mutate { add_field => { "[@metadata][ObjectCode]" => "%{[eventHeader][applicationId]}%{[eventHeader][objectCode]}"} }
mutate { add_field => { "[@metadata][action]" => "%{[event][action]}"} }
mutate { add_field => { "[@metadata][applicationId]" => "%{[eventHeader][applicationId]}"} }
mutate { add_field => { "[@metadata][itemId]" => "%{[eventData][itemId]}"} }
mutate { add_field => { "[@metadata][isDeleted]" => "%{[eventData][isDeleted]}"} }
mutate { gsub => ["[@metadata][action]","INSERT","index"] }
mutate { gsub => ["[@metadata][action]","Insert","index"] }
mutate { gsub => ["[@metadata][action]","UPDATE","update"] }
mutate { gsub => ["[@metadata][action]","Update","update"] }
mutate { gsub => ["[@metadata][action]","DELETE","delete"] }
mutate { gsub => ["[@metadata][action]","Delete","delete"] }
mutate { remove_field => "[logstash]" }
mutate { remove_field => "[metadata]" }
mutate { remove_field => "[eventHeader]" }
mutate { remove_field => "[event]" }
json{
source => "message"
}

ruby {
code => '
details_hash = event.get("eventData")
return if details_hash.nil?
details_hash.each do |k, v|
event.set(k,v)
end
event.remove("eventData")
hash = event.to_hash
hash.each {|k,v|
data = event.get(k)
return if data.nil?
event.set(k,v)
}'
}
json{ source=>"message" }
}
input {
http_poller {
urls => {
soap_request => {
method => post
url => "https://dm-dev-authenticationservice-serviceapps-use-rg.azurewebsites.net/api/v1/Token"
headers => {
"Content-Type" => "application/json"
}
body =>
'{"ClientId":"%[@metadata][itemId]"}'
}
}
request_timeout => 60
# Supports "cron", "every", "at" and "in" schedules by rufus scheduler
schedule => { cron => "* * * * * UTC"}
codec => "json"
# A hash of request metadata info (timing, response headers, etc.) will be sent here
metadata_target => "http_poller_metadata"
}
}
output{
if [@metadata][applicationId] == "62"
{
elasticsearch {
hosts => "http://172.20.188.15:9200"
user => "elastic"
password => "changeme"
index => "%{[@metadata][index]}"
document_id => "%{[@metadata][ObjectCode]}"
action => "%{[@metadata][action]}"
}
}
stdout { codec => rubydebug { metadata => true} }
}
this is my config file I want [@metadata][itemId] needs to be replcae in body section
body =>
'{"ClientId":"%[@metadata][itemId]"}'

like body =>
'{"ClientId":"123"}'

2

Please use formatting so that your question is readable.

I think you just have to change

'{"ClientId":"%[@metadata][itemId]"}'

to

'{"ClientId":"%{[@metadata][itemId]}"}'
3

Hi Thanks,

for quick responce still its not working for me,
its showing below content
"body" => "{"ClientId":"%{[@metadata][itemId]}"}",
can we use filter data inside input?

4

Can you please format the config in your question?
Cannot figure out where the config starts and ends.
Start and end config block with 3 backticks (`)

7 
input{
kafka {
	bootstrap_servers => "172.20.188.11:9092,172.20.188.31:9092,172.20.188.30:9092"
	topics => ["DS_Test"]
	decorate_events => true
	group_id => "ds_test"
	codec => json
	max_poll_records => "100"
	consumer_threads => 2
	session_timeout_ms => "30000"
	auto_offset_reset => "earliest"
	client_id => "logstash-ingestion"
 }
}

filter {
	mutate { remove_field => "@Version" }
	mutate { remove_field => "@timestamp" }
	mutate { add_field => { "[@metadata][index]" => "%{[eventHeader][bpcName]}%{[eventHeader][applicationId]}"} }
	mutate { add_field => { "[@metadata][ObjectCode]" => "%{[eventHeader][applicationId]}%{[eventHeader][objectCode]}"} }
	mutate { add_field => { "[@metadata][action]" => "%{[event][action]}"} }
	mutate { add_field => { "[@metadata][applicationId]" => "%{[eventHeader][applicationId]}"} }
	mutate { add_field => { "[@metadata][itemId]" => "%{[eventData][itemId]}"} }
	mutate { add_field => { "[@metadata][isDeleted]" => "%{[eventData][isDeleted]}"} }
	mutate { gsub => ["[@metadata][action]","INSERT","index"] }
	mutate { gsub => ["[@metadata][action]","Insert","index"] }
	mutate { gsub => ["[@metadata][action]","UPDATE","update"] }
	mutate { gsub => ["[@metadata][action]","Update","update"] }
	mutate { gsub => ["[@metadata][action]","DELETE","delete"] }
	mutate { gsub => ["[@metadata][action]","Delete","delete"] }
	mutate { remove_field => "[logstash]" }
	mutate { remove_field => "[metadata]" }
	mutate { remove_field => "[eventHeader]" }
	mutate { remove_field => "[event]" }

ruby {
	code => '
		details_hash = event.get("eventData")
		return if details_hash.nil?
		details_hash.each do |k, v|
		event.set(k,v)
		end
		event.remove("eventData")
		hash = event.to_hash
		hash.each {|k,v|
		data = event.get(k)
		return if data.nil?
		event.set(k,v)
		}'
	}
json{ source=>"message" }
}
input {
	http_poller {
			urls => {
				soap_request => {
					method => post
					url => "https://dm-dev-authenticationservice-serviceapps-use-rg.azurewebsites.net/api/v1/Token"
					headers => {
					"Content-Type" => "application/json"
					}
					body =>
					'{"ClientId":"%[@metadata][itemId]"}'
				  }
			}
		request_timeout => 60
		schedule => { cron => "* * * * * UTC"}
		codec => "json"	
		metadata_target => "http_poller_metadata"
	}
}
output{
	if [@metadata][applicationId] == "62"
	{
		elasticsearch {
			hosts => "http://172.20.188.15:9200"
			user => "elastic"
			password => "changeme"
			index => "%{[@metadata][index]}"
			document_id => "%{[@metadata][ObjectCode]}"
			action => "%{[@metadata][action]}"
		}
	}
stdout { codec => rubydebug { metadata => true} }
}

please find formatted config
8

I want to assign value to ClientId which is [@metadata][itemId] value .
want to understand can we assign filter value inside input

9

I do not think this will work. The url (including the options hash which contains the body option) is passed to the Manticore library that logstash uses. logstash does not sprintf the option before passing it, so sprintf references like this will not get expanded.

10

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.