My complete logstash conf file, please suggest where i am going wrong as there is no change after adding push_map_as_event_on_timeout:
input{
beats{
port => "5043"
}
}
filter{
if [log_type] == "fullactivelogs" {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:Activity-Time}%{SPACE}USERID=%{QUOTEDSTRING:USERID}%{SPACE}IP=\"%{IP:IP}\"%{SPACE}SESSION=%{QUOTEDSTRING}%{SPACE}HOST=%{QUOTEDSTRING:HOST}%{SPACE}HOMEDIR=%{QUOTEDSTRING:HOMEDIR}%{SPACE}SITE=%{QUOTEDSTRING:SITE}%{SPACE}METHOD=%{QUOTEDSTRING:METHOD}%{SPACE}HTTP_USER_AGENT=%{QUOTEDSTRING:HTTPAGENT}%{SPACE}MENU=%{QUOTEDSTRING:MENU}%{SPACE}COMPONENT=%{QUOTEDSTRING:COMPONENT}%{SPACE}PAGE=%{QUOTEDSTRING:PAGE}%{SPACE}KEYS=%{QUOTEDSTRING:KEYS}%{SPACE}ACTION=%{QUOTEDSTRING:ACTION}%{SPACE}URL=%{QUOTEDSTRING:URL}%{SPACE}DEVICETYPE=%{QUOTEDSTRING:DEVICETYPE}%{SPACE}RENDERMODE=%{QUOTEDSTRING:RENDERMODE}%{SPACE}PHYSICALHOST=%{QUOTEDSTRING:PHYSICALHOST}%{SPACE}WEBAPPDOMAIN=%{QUOTEDSTRING:WEBAPPDOMAIN}%{SPACE}ELAPSEDSECONDS=\"%{NUMBER:ELAPSEDTIMEINSECONDS:float}\"" }}
geoip{
source => "IP"
fields => ["longitude","latitude"]
target => "client.geo.location"
}
mutate { gsub => [ "Activity-Time", ",.*", "" ] }
}
if [log_type] == "pages" {
grok{
match => {"message" => "%{TIMESTAMP_ISO8601:Activity-Time},PAGE=%{QUOTEDSTRING:PAGE},DESCRIPTION=%{QUOTEDSTRING:PAGEDESCRIPTION},USAGE=%{QUOTEDSTRING:PAGEUSAGE},PPI_LEVEL=%{QUOTEDSTRING:PPI_LEVEL}"}
}
mutate { gsub => [ "Activity-Time", ",.*", "" ] }
}
aggregate{
task_id => "%{PAGE}"
code => "
map['PAGEDESCRIPTION'] ||= event.get('PAGEDESCRIPTION')
map['HTTPAGENT'] ||= event.get('HTTPAGENT')
"
map_action => "create_or_update"
timeout => 10
push_map_as_event_on_timeout => true
}
}
output{
elasticsearch {
hosts => ["10.0.62.184:9200"]
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
template_name => "testtemplate"
}
}
Please guide if aggregate is correct?
Is the placement and syntax correct ? Am i missing anything