Hi,
In previous i was using the version 2.3 for logstash in that aggregation was worked fine. Few days back i changed my ELK into latest version. Now im using logstash 5.0.0 in that aggregate(2.5.1) is not working anyone can help me out of it, Please find the below code for reference
input {
beats{
port => 10523
}
}
filter
{
if [type] == "ddoa_req" {
xml{
source => "message"
store_xml => false
remove_namespaces => true
xpath => [
"//context/correlation/dealerID/text()","dlr_dlrCode"
]
}
mutate {
gsub => [
"request_fileName","-",""
]
}
mutate{
remove_field => ["message","type"]
}
aggregate {
task_id => "%{request_fileName}"
code => "event.set('response_fileName', map['response_fileName'])
event.set('app_res_releaseID', map['app_res_releaseID'])
event.set('app_res_creatorNameCode', map['app_res_creatorNameCode'])
event.set('app_res_senderNameCode', map['app_res_senderNameCode'])
event.set('app_res_sysVersion', map['app_res_sysVersion'])
event.set('app_res_creationDateTime', map['app_res_creationDateTime'])
event.set('app_res_bodID', map['app_res_bodID'])
event.set('app_res_destinationName', map['app_res_destinationName'])
event.set('response_desc', map['response_desc'])
event.set('response_status', map['response_status'])
event.set('response_reason', map['response_reason'])"
map_action => "update"
end_of_task => true
}
fingerprint {
source => ["request_fileName"]
target => "fingerprint"
key => "78787878"
method => "SHA1"
concatenate_sources => true
}
}
if [type] == "ddoa_res" {
xml{
source => "message"
store_xml => false
remove_namespaces => true
xpath => [
"//context/correlation/fileName/text()","response_fileName",
"//body/ProcessMessageResponse/payload/content/ConfirmBOD/@releaseID","app_res_releaseID",
"//body/ProcessMessageResponse/payload/content/ConfirmBOD/ApplicationArea/Sender/CreatorNameCode/text()","app_res_creatorNameCode",
"//body/ProcessMessageResponse/payload/content/ConfirmBOD/ApplicationArea/Sender/SenderNameCode/text()","app_res_senderNameCode",
"//body/ProcessMessageResponse/payload/content/ConfirmBOD/ApplicationArea/Sender/SystemVersion/text()","app_res_sysVersion",
"//body/ProcessMessageResponse/payload/content/ConfirmBOD/ApplicationArea/CreationDateTime/text()","app_res_creationDateTime",
"//body/ProcessMessageResponse/payload/content/ConfirmBOD/ApplicationArea/BODID/text()","app_res_bodID",
"//body/ProcessMessageResponse/payload/content/ConfirmBOD/ApplicationArea/Destination/DestinationNameCode/text()","app_res_destinationName",
"//*[local-name()='Description']/text()","response_desc",
"//*[local-name()='ReasonCode']/text()","response_reason"
]
}
grok{
match =>{"message" => ".?(?<response_status>(BODSuccessMessage))"}
}
grok{
match =>{"message" => ".?(?<response_status>(BODFailureMessage))"}
}
translate {
field => "response_status"
destination => "response_status"
override => true
dictionary => ["BODSuccessMessage","Success",
"BODFailureMessage","Failure"]
}
mutate {
gsub => [
"response_fileName","-",""
]
}
mutate{
remove_field => ["message","type","path"]
}
aggregate {
task_id => "%{response_fileName}"
code => "map['response_fileName'] = event.get('response_fileName')
map['app_res_releaseID'] = event.get('app_res_releaseID')
map['app_res_creatorNameCode'] = event.get('app_res_creatorNameCode')
map['app_res_senderNameCode'] = event.get('app_res_senderNameCode')
map['app_res_sysVersion'] = event.get('app_res_sysVersion')
map['app_res_creationDateTime'] = event.get('app_res_creationDateTime')
map['app_res_bodID'] = event.get('app_res_bodID');
map['app_res_destinationName'] = event.get('app_res_destinationName')
map['response_desc'] = event.get('response_desc')
map['response_status'] = event.get('response_status')
map['response_reason'] = event.get('response_reason')"
map_action => "create"
}
fingerprint {
source => ["response_fileName"]
target => "fingerprint"
key => "78787878"
method => "SHA1"
concatenate_sources => true
}
}
}
output{
elasticsearch {
index => "logstash-dd.ddoa_req_log_v1"
hosts => ["localhost:9200"]
document_id => "%{fingerprint}" # !!! prevent duplication
}
stdout {
codec => rubydebug
}
}