Need to count the number of occurrence in IIS logs, used logstash filter aggregate function and tagged it.
There is two part on actual raw data should forward to elasticsearch and aggregate should data should be in influx db .i had differentiate with two tags though only aggregate function data is forwarding to elasticsearch instead of raw data . Here is my code..Could you please help here..
if [type] == "iislog" {
grok { match => { "message" => "%{ WORD : tag_tec}"}
}
json {
source => "message"
}
grok {
match => ["tag_tec", "IIS"]
add_tag => "IIS"
}
grok {
match => ["tag_tec", "IIS"]
add_tag => "IISIn"
}
grok {
match => ["tag_tec","WINDOWS"]
add_tag => "WINDOWS"
}
grok
{
match => ["tag_tec","AD"]
add_tag => "AD"
}
grok
{
match => ["tag_tec", "UNIX"]
add_tag => "UNIX"
}
grok
{
match => ["tag_tec" ,"LINUX"]
add_tag => "LINUX"
}
grok
{
match => ["tag_tec" ,"APACHE"]
add_tag => "APACHE"
}
if "IIS" in [tags]
{
grok {
match => {"message" => "%{TIMESTAMP_ISO8601:EventReceivedTime} %{WORD:SourceModuleName} %{WORD:SourceModuleType} %{TIMESTAMP_ISO8601:EventTime} %{WORD:date} %{WORD:time} %{IPORHOST:s_ip} %{WORD:cs_method} %{URIPATH:cs_uri_stem} %{NOTSPACE:cs_uri_query} %{NUMBER:s_port} %{NOTSPACE:cs_username} %{IPORHOST:c_ip} %{NOTSPACE:cs_User_Agent} %{NOTSPACE:cs_Referer} %{NUMBER:sc_status} %{NUMBER:sc_substatus} %{NUMBER:sc_win32_status} %{NUMBER:timetaken} %{WORD:tag_app} %{WORD:tag_cus} %{WORD:tag_loc} %{WORD:tag_tec} %{WORD:Hostname} %{WORD:tag_CI} "}
}
mutate {
remove_tag => [ "IISIn","WINDOWS","UNIX","LINUX","APACHE" ]
}
}
if "IISIn" in [tags]
{
grok {
match => {"message" => "%{TIMESTAMP_ISO8601:EventReceivedTime} %{NUMBER:sc_status} %{WORD:tag_app} %{WORD:tag_cus} %{WORD:tag_loc} %{WORD:tag_tec} %{WORD:Hostname} %{WORD:tag_CI} "}
}
grok {
match => ["sc_status","[2]\d\d"]
#add_field => {"application" => "BOOKMYFLOWER"}
add_tag => "success"
}
grok {
match => ["sc_status","[3,4,5]\d\d"]
#add_field => {"application" => "BOOKMYFLOWER"}
add_tag => "failure"
}
if "success" in [tags]
{
mutate{ add_field => {"status" => "Success"}}
aggregate
{
task_id => "%{tag_app} on %{tag_app} %{tag_cus}"
code => "
map['sc_status'] ||= []
require 'matrix'
event.to_hash.each do |key,value|
map[key] = value unless map.has_key?(key)
map[key] << value if map[key].is_a?(Array)and !value.is_a?(Array)
map['count']= map['sc_status'].each_with_object(Hash.new(0)) { |o, h| h[o] += 1;h }
map['responses']=[]
map['errorcount']=[]
map['count'].each do |key,value|
map['responses']<< key
map['errorcount']<< value
end
end
event.cancel()"
push_previous_map_as_event => true
periodic_flush => true
timeout => 3
add_tag => ['aggregated_IIS_S']
}
if "aggregated_IIS_S" not in [tags] {
drop {}
}
}
if "failure" in [tags]
{
mutate{ add_field => {"status" => "Failure"}}
aggregate
{
task_id => "%{tag_tec} on %{tag_cus}"
code => "
map['sc_status'] ||= []
require 'matrix'
event.to_hash.each do |key,value|
map[key] = value unless map.has_key?(key)
map[key] << value if map[key].is_a?(Array)and !value.is_a?(Array)
map['count']= map['sc_status'].each_with_object(Hash.new(0)) { |o, h| h[o] += 1;h }
map['responses']=[]
map['errorcount']=[]
map['count'].each do |key,value|
map['responses']<< key
map['errorcount']<< value
end
end
event.cancel() "
push_previous_map_as_event => true
periodic_flush => true
timeout => 3
add_tag => ['IIS_F']
}
if "IIS_F" not in [tags] {
drop {}
}
}
}