Calculate Elapsed time in logstash pipe


(Arun Prakash) #1

Hi All,

Please help me out to find the elapsed time from pipe started between pipe ended and its status whether it is completed or not.

my code is as below,

input {
jdbc {
jdbc_connection_string => "jdbc:oracle:thin:@01hw791865.India.TCS.com:1521/XE"
jdbc_user => "SYSTEM"
jdbc_password => "Vijaya"
jdbc_driver_library => "C:\Users\915360.m2\repository\oracle\ojdbc6\11.2.0.3\ojdbc6-11.2.0.3.jar"
jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
statement => "SELECT to_char(sysdate,'YYYYMMDDHH') as batchid, CUSTOMER.CUSTOMERID,CUSTOMER.FIRSTNAME,CUSTOMER.LASTNAME,CUSTOMER.DOB,' [ '||LISTAGG('{ '||chr(34)||'agentid'||chr(34)||' : '||nvl(AGENTDETAILS.AGENTID,-1)||', '||chr(34)||'agentfn'||chr(34)||' : '||chr(34)||nvl(AGENTDETAILS.FIRSTNAME,'-1')||chr(34)||', '||chr(34)||'agentln'||chr(34)||' : '||chr(34)||nvl(AGENTDETAILS.LASTNAME,'-1')||chr(34)||', '||chr(34)||'agencyname'||chr(34)||' : '||chr(34)||nvl(AGENTDETAILS.AGENCYNAME,'-1')||chr(34)||'}' ,',') within group (order by agentdetails.agentid) || ' ] ' as agents_unparsed,replace(' [ '||RTRIM(REGEXP_REPLACE(LISTAGG('{ '||chr(34)||'policyno'||chr(34)||' : '||chr(34)||policydetails.policyno||chr(34)||', '||chr(34)||'policytype'||chr(34)||' : '||chr(34)||policydetails.policytype||chr(34)||' }' ,';') within group (order by policydetails.policyno),'([^;]*)(;\1)+($|;)','\1\3'),';') || ' ] ',';',',') as policy_unparsed FROM MV_PROMPT_LTAM.CUSTOMER LEFT OUTER JOIN MV_PROMPT_LTAM.CUSTOMERAGENT ON CUSTOMERAGENT.CUSTOMERID = CUSTOMER.CUSTOMERID LEFT OUTER JOIN MV_PROMPT_LTAM.AGENTDETAILS ON CUSTOMERAGENT.AGENTID = AGENTDETAILS.AGENTID LEFT OUTER JOIN MV_PROMPT_LTAM.CUSTOMERPOLICY ON CUSTOMER.CUSTOMERID = CUSTOMERPOLICY.CUSTOMERID LEFT OUTER JOIN MV_PROMPT_LTAM.POLICYDETAILS ON CUSTOMERPOLICY.POLICYNO = POLICYDETAILS.POLICYNO GROUP BY CUSTOMER.CUSTOMERID,CUSTOMER.FIRSTNAME,CUSTOMER.LASTNAME,CUSTOMER.DOB"
}
}
filter {
json {
source => "agents_unparsed"
target => "agents"
}
json {
source => "policy_unparsed"
target => "policy"
}
mutate{
remove_field => ["agents_unparsed","policy_unparsed"]
}

Measures the execution time of system1

elapsed {
unique_id_field => "batchid"
start_tag => "system1Enter"
end_tag => "system1Exit"
new_event_on_match => true
add_tag => ["in1"]
}

Measures the time between system1 and system2

elapsed {

unique_id_field => "batchid"

start_tag => "system1Exit"

end_tag => "system2Enter"

new_event_on_match => true

add_tag => ["1->2"]

}

# Measures the execution time of system2

elapsed {

unique_id_field => "batchid"

start_tag => "system2Enter"

end_tag => "system2Exit"

new_event_on_match => true

add_tag => ["in2"]

}

Records the execution time of system1

if "in1" in [tags] and "elapsed" in [tags] {
aggregate {
task_id => "%{batchid}"
code => "map['report'] = (event.get('elapsed_time')*1000).to_i"
map_action => "create"
}
}

Records the time between system1 and system2

if "1->2" in [tags] and "elapsed" in [tags] {

aggregate {

task_id => "%{batchid}"

code => "map['report'] << (event.get('elapsed_time')*1000).to_i"

map_action => "update"

}

}

# Records the execution time of system2

if "in2" in [tags] and "elapsed" in [tags] {

aggregate {

task_id => "%{batchid}"

code => "map['report'] << (event.get('elapsed_time')*1000).to_i"

map_action => "update"

end_of_task => true

}

}

}
#if [performance][event_type] == "enter" {

mutate { add_tag => ["taskstarted"] }

#} else if [performance][event_type] == "exit" {

mutate { add_tag => ["taskterminated"] }

#}
#elapsed {

start_tag => "taskstarted"

end_tag => "taskterminated"

unique_id_field => "batchid"

timeout => 10000

new_event_on_match => false

#}

elapsed {

start_tag => "taskStarted"

end_tag => "taskTerminated"

unique_id_field => "batchid"

}

elapsed {

start_tag => "taskstarted"

end_tag => "taskended"

unique_id_field => "batchid"

timeout => seconds

new_event_on_match => true

}

#}
#filter {
#ruby {
#code => "
#require 'json'
#agents_value = JSON.parse(event.get('agents').to_s)
#event.set('agents',agents_value)
#"
#}
#}
output {
mongodb {
uri => "mongodb://localhost:27017/"
database => "customer"
collection => "customer_nested"
isodate => true
}
}


(system) #2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.