How to send http_poller response data in elastic search

Elastic Stack Logstash
1

Hi Guys,

can someone will help me to add http_poller response in elastic.

my config file
input{
kafka {
bootstrap_servers => "172.20.188.11:9092,172.20.188.31:9092,172.20.188.30:9092"
topics => ["DS_Test"]
decorate_events => true
group_id => "ds_test"
codec => json
max_poll_records => "100"
consumer_threads => 2
session_timeout_ms => "30000"
auto_offset_reset => "earliest"
client_id => "logstash-ingestion"
}
}
input {
http_poller {
urls => {
soap_request => {
method => post
url => "https://dm-dev-authenticationservice-serviceapps-use-rg.azurewebsites.net/api/v1/Token"
headers => {
"Content-Type" => "application/json"
}
body => '{"ClientId":"sdmtSG9BOap6aLn5oxoaCrfLHQCIUMLF","ClientSecret":"SqN4ywxCu6ylvn7ecoViEWB2NlUk9hup0eBn5VquT0XutSXCh6VKXC82dREeGIi2","Audience":"https://InventoryService/"}'
}
}
request_timeout => 60
# Supports "cron", "every", "at" and "in" schedules by rufus scheduler
schedule => { cron => "* * * * * UTC"}
codec => "json"
# A hash of request metadata info (timing, response headers, etc.) will be sent here
metadata_target => "http_poller_metadata"
}
}
filter {
mutate { remove_field => "@Version" }
mutate { remove_field => "@timestamp" }
mutate { add_field => { "[@metadata][index]" => "%{[eventHeader][bpcName]}%{[eventHeader][applicationId]}"} }
mutate { add_field => { "[@metadata][ObjectCode]" => "%{[eventHeader][applicationId]}%{[eventHeader][objectCode]}"} }
mutate { add_field => { "[@metadata][action]" => "%{[event][action]}"} }
mutate { add_field => { "[@metadata][applicationId]" => "%{[eventHeader][applicationId]}"} }
mutate { add_field => { "[@metadata][itemId]" => "%{[eventData][itemId]}"} }
mutate { add_field => { "[@metadata][isDeleted]" => "%{[eventData][isDeleted]}"} }
mutate { gsub => ["[@metadata][action]","INSERT","index"] }
mutate { gsub => ["[@metadata][action]","Insert","index"] }
mutate { gsub => ["[@metadata][action]","UPDATE","update"] }
mutate { gsub => ["[@metadata][action]","Update","update"] }
mutate { gsub => ["[@metadata][action]","DELETE","delete"] }
mutate { gsub => ["[@metadata][action]","Delete","delete"] }
mutate { remove_field => "[logstash]" }
mutate { remove_field => "[metadata]" }
mutate { remove_field => "[eventHeader]" }
mutate { remove_field => "[event]" }
json{
source => "message"
}
mutate{
add_field => { "status" => "%{[http_poller_metadata][code]}" }
}
mutate {
copy => {
"status" => "statuscode"
}
}
ruby {
code => '
details_hash = event.get("eventData")
return if details_hash.nil?
details_hash.each do |k, v|
event.set(k,v)
end
event.remove("eventData")
hash = event.to_hash
hash.each {|k,v|
data = event.get(k)
return if data.nil?
event.set(k,v)
}'
}
json{ source=>"message" }
}
output{
if [@metadata][applicationId] == "62"
{
elasticsearch {
hosts => "http://172.20.188.15:9200"
user => "elastic"
password => "changeme"
index => "%{[@metadata][index]}"
document_id => "%{[@metadata][ObjectCode]}"
action => "%{[@metadata][action]}"
}
}
stdout { codec => rubydebug { metadata => true} }
}

My resposne:-

{
"normalizedText" => "",
"@metadata" => {
"action" => "index",
"index" => "flowersfoods_62",
"kafka" => {
"key" => nil,
"offset" => 226,
"consumer_group" => "ds_test",
"topic" => "DS_Test",
"partition" => 0,
"timestamp" => -1
},
"ObjectCode" => "62_125",
"applicationId" => "62",
"itemId" => "125",
"isDeleted" => "%{[eventData][isDeleted]}"
},
"IndexedOn" => "2020-06-03T15:56:42.2120593+05:30",
"type" => "AS",
"nounModifier" => "",
"statuscode" => "%{[http_poller_metadata][code]}",
"itemId" => 125,
"field" => "itemId",
"softDelete" => "Y",
"appId" => 36,
"@version" => "1",
"contactcode" => 0,
"text" => 90,
"status" => "%{[http_poller_metadata][code]}"
}
{
"statuscode" => "200",
"expiresIn" => 86400,
"@metadata" => {
"action" => "%{[event][action]}",
"index" => "%{[eventHeader][bpcName]}%{[eventHeader][applicationId]}",
"ObjectCode" => "%{[eventHeader][applicationId]}%{[eventHeader][objectCode]}",
"applicationId" => "%{[eventHeader][applicationId]}",
"itemId" => "%{[eventData][itemId]}",
"isDeleted" => "%{[eventData][isDeleted]}"
},
"@version" => "1",
"http_poller_metadata" => {
"request" => {
"headers" => {
"Content-Type" => "application/json"
},
"method" => "post",
"body" => "{"ClientId":"sdmtSG9BOap6aLn5oxoaCrfLHQCIUMLF","ClientSecret":"SqN4ywxCu6ylvn7ecoViEWB2NlUk9hup0eBn5VquT0XutSXCh6VKXC82dREeGIi2","Audience":"https://InventoryService/"}",
"url" => "https://dm-dev-authenticationservice-serviceapps-use-rg.azurewebsites.net/api/v1/Token"
},
"response_headers" => {
"date" => "Wed, 03 Jun 2020 10:27:00 GMT",
"set-cookie" => [
[0] "ARRAffinity=58e912a579bcbdcd4d39cdfe9f22b26a8257c0436fbb6567ba405f84999f917f;Path=/;HttpOnly;Domain=dm-dev-authenticationservice-serviceapps-use-rg.azurewebsites.net",
[1] "ARRAffinity=e3f151f06fbd0f78471b3feb8433ea3f4f2a489a9d92ad613a1ca01018b4cf04;Path=/;HttpOnly;Domain=dm-dev-authenticationservice-serviceapps-use-rg.azurewebsites.net"
],
"request-context" => "appId=cid-v1:7d56ed69-504b-4a83-bc46-7bb3b515ed95",
"vary" => "Accept-Encoding",
"content-type" => "application/json; charset=utf-8"
},
"code" => 200,
"response_message" => "OK",
"times_retried" => 0,
"runtime_seconds" => 0.977467,
"name" => "soap_request",
"host" => "SEADEVDMLOGSTASH7x2"
},
"accessToken" => "VdZG9Nd+CMJA6UVBAxPSE5eFdNt3lfJc6CkbmWs1m20NhpAxlaw41lBMHByhfwBlzuwLAdal9wRVZbaQm9+F+k2PW2A60bufj631K1OEcrndEtrfrIP0kdPKqTDzcby0eSWKoy2Oi6CburrtuTFsYyRCP71Go+Url9yQ1o812dXPzV2JfhwLeEcFGTSX7ktsRm1N9vfPiEeTHdM1Q04lMUvPu30dvI+AEyN9npdyauz5JQmTUWkwfLR5nNJiUwwdHGkdrMw7U+LTUwiNGFKx3MxYdbOxoD5ZxwRobF3TcYqeQ35xT4kbeVHqjy2yKgGySh5sjiKdKfTisT4tQZ+X3rQA34HdorE4zK3ioQzsrlPSSm1KSmdomIOqpvg9UEnFoXmwlPsukjLX5mcx5BsHpDLUkhFuIwSvr98QhN9zHIPMWF2q26neECb4D6cKhzMpjFKIlNu4aTORij5tL3gw7tzWWKkoRjELTHWhrpa4SMizK8eaNVWnxgwbfplO9T510Esvz24JI8h1KAP387TTHlxL1IHAZK9MCgo2QJCwSZRCPdv9H0Ucf8vNwAdlkb+KuxNDZCjzTX31pi3gsku1FVevtsEBTPMYrpyksVvAKK2i07Vnc4LXpdoLSBY8mt94Pgyoosr2eYLf2z0HRh7CMrf+0elG50rnhpShWHAwvshuxDgepJrnYkFOLKDMa5SxyABLnLeaCAhsZwrUOdCAUORel8qUda9xkFb3wTgkoR4eph5xKqAftzhgjGECxnt5XTF0wmMdZKlRk9M0y6gHYH1IjTdB8AIS6sEBhPiONMFHQWaq6hoVO9yQ6BBXkHwClLD2sPIhmq7mK5hKk1nrpXD95RuepeSm+Y8BaoDUxmlnC9gynyJ+jBtlLt8bfoKbPOd9sPKe5LtxC2oeabB465mlQJerWbekUwsbfjNssOyixpI4H5JCij8a4kI5XoTleq2nYdWsRnDTDvxn7xAW6cOPkIVb8hNxbkotVM9IdkboF4Ac1ce2XS1PeN03387UVFzis+sOLprWyG7JmCVQjqaluYzzeWnxaskJn2NBduy3Wz1L0gINzEv+siajVABwBWlzo432GfCbSCU+T+3gd8H9MtFtHWOINqQlqbvALA3MGwfqOtZzUIPNMREx3zxCCdNpDGMdExxXNvq5UhrLMgLR6ZRQuiB7fQnrArLHifBDBnbBnoBGTqbFzS2fCKdji0XxrKfxNKQ2X+X2H718JOciE4SeuEBPuYeO8rhxBDBZPvOZKy2zQ9KqzNRdH9nIZrCXkr1MFrrgi1FioEWfEhirOimAXeYUOlsYlCHm",
"tokenType" => "Bearer",
"status" => "200"
}

this is the respose after debug

if you see I want to use "status" => "200" add it to the elastic search but as per current config file

it inserting this information "status" : "%{[http_poller_metadata][code]}", in elastic search.

2

Is there any event or method which will trigger in logstash/filter after http_poller response because am getting 2 object and it inserting value of 1st object but my 2nd object response is the http_poller response i've to use the "status" from that object and add/update in elastic search

3

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.