Hi there,
I got a Problem with my current Logstash configuration.
I am trying to write a config for the Sophos UTM logs, the fields gets extracted like I want but every time there is
the same value 3 times in the specific fields....
Any ideas why?
My configuration:
filter {
...
if "ulogd" in [message]{
grok {
break_on_match => false
match => [
"message", "<%{INT:fw_id}>(?%{YEAR}:%{MONTHNUM}:%{MONTHDAY}-%{HOUR}:%{MINUTE}:%{SECOND}) %{HOSTNAME:logsource} %{WORD:program}[%{INT:pid}]: id="%{INT:id}" severity="%{WORD:severity}" sys="%{WORD:sys}" sub="%{WORD:sub}" name="%{DATA:name}" action="%{WORD:action}" fwrule="%{INT:fwrule}" outitf="%{NOTSPACE:outitf}" mark="%{DATA:mark}" app="%{DATA:app}" srcmac="%{MAC:srcmac}" srcip="%{IPV4:source_ip}" dstip="%{IPV4:destination_ip}" proto="%{WORD:protocol}" length="%{INT:length}" tos="%{DATA:tos}" prec="%{DATA:prec}" ttl="%{INT:ttl}" srcport="%{INT:srcport}" dstport="%{INT:dstport}" ", "message", "<%{INT:fw_id}>(?%{YEAR}:%{MONTHNUM}:%{MONTHDAY}-%{HOUR}:%{MINUTE}:%{SECOND}) %{HOSTNAME:logsource} %{WORD:program}[%{INT:pid}]: id="%{INT:id}" severity="%{WORD:severity}" sys="%{WORD:sys}" sub="%{WORD:sub}" name="%{DATA:name}" action="%{WORD:action}" fwrule="%{INT:fwrule}" initf="%{NOTSPACE:initf}" outitf="%{NOTSPACE:outitf}" srcmac="%{MAC:srcmac}" dstmac="%{MAC:dstmac}" srcip="%{IPV4:source_ip}" dstip="%{IPV4:destination_ip}" proto="%{WORD:protocol}" length="%{INT:length}" tos="%{DATA:tos}" prec="%{DATA:prec}" ttl="%{INT:ttl}" srcport="%{INT:srcport}" dstport="%{INT:dstport}" ",
"message", "<%{INT:fw_id}>(?%{YEAR}:%{MONTHNUM}:%{MONTHDAY}-%{HOUR}:%{MINUTE}:%{SECOND}) %{HOSTNAME:logsource} %{WORD:program}[%{INT:pid}]: id="%{INT:id}" severity="%{WORD:severity}" sys="%{WORD:sys}" sub="%{WORD:sub}" name="%{DATA:name}" action="%{WORD:action}" fwrule="%{INT:fwrule}" initf="%{NOTSPACE:initf}" outitf="%{NOTSPACE:outitf}" srcmac="%{MAC:srcmac}" srcip="%{IPV4:source_ip}" dstip="%{IPV4:destination_ip}" proto="%{WORD:protocol}" length="%{INT:length}" tos="%{DATA:tos}" prec="%{DATA:prec}" ttl="%{INT:ttl}" srcport="%{INT:srcport}" dstport="%{INT:dstport}" ","message", "<%{INT:fw_id}>(?%{YEAR}:%{MONTHNUM}:%{MONTHDAY}-%{HOUR}:%{MINUTE}:%{SECOND}) %{HOSTNAME:logsource} %{WORD:program}[%{INT:pid}]: id="%{INT:id}" severity="%{WORD:severity}" sys="%{WORD:sys}" sub="%{WORD:sub}" name="%{DATA:name}" action="%{WORD:action}" fwrule="%{INT:fwrule}" outitf="%{NOTSPACE:outitf}" srcmac="%{MAC:srcmac}" srcip="%{IPV4:source_ip}" dstip="%{IPV4:destination_ip}" proto="%{WORD:protocol}" length="%{INT:length}" tos="%{DATA:tos}" prec="%{DATA:prec}" ttl="%{INT:ttl}" srcport="%{INT:srcport}" dstport="%{INT:dstport}" ","message", "<%{INT:fw_id}>(?%{YEAR}:%{MONTHNUM}:%{MONTHDAY}-%{HOUR}:%{MINUTE}:%{SECOND}) %{HOSTNAME:logsource} %{WORD:program}[%{INT:pid}]: id="%{INT:id}" severity="%{WORD:severity}" sys="%{WORD:sys}" sub="%{WORD:sub}" name="%{DATA:name}" action="%{WORD:action}" fwrule="%{INT:fwrule}" initf="%{NOTSPACE:initf}" outitf="%{NOTSPACE:outitf}" srcmac="%{MAC:srcmac}" dstmac="%{MAC:dstmac}" srcip="%{IPV4:source_ip}" dstip="%{IPV4:destination_ip}" proto="%{WORD:protocol}" length="%{INT:length}" tos="%{DATA:tos}" prec="%{DATA:prec}" ttl="%{INT:ttl}" srcport="%{INT:srcport}" dstport="%{INT:dstport}" ","message", "<%{INT:fw_id}>(?%{YEAR}:%{MONTHNUM}:%{MONTHDAY}-%{HOUR}:%{MINUTE}:%{SECOND}) %{HOSTNAME:logsource} %{WORD:program}[%{INT:pid}]: id="%{INT:id}" severity="%{WORD:severity}" sys="%{WORD:sys}" sub="%{WORD:sub}" name="%{DATA:name}" action="%{WORD:action}" fwrule="%{INT:fwrule}" initf="%{NOTSPACE:initf}" outitf="%{NOTSPACE:outitf}" srcmac="%{MAC:srcmac}" dstmac="%{MAC:dstmac}" srcip="%{IPV4:source_ip}" dstip="%{IPV4:destination_ip}" proto="%{WORD:protocol}" length="%{INT:length}" tos="%{DATA:tos}" prec="%{DATA:prec}" ttl="%{INT:ttl}" srcport="%{INT:srcport}" dstport="%{INT:dstport}" tcpflags="%{DATA:tcpflags}" ","message", "<%{INT:fw_id}>(?%{YEAR}:%{MONTHNUM}:%{MONTHDAY}-%{HOUR}:%{MINUTE}:%{SECOND}) %{HOSTNAME:logsource} %{WORD:program}[%{INT:pid}]: id="%{INT:id}" severity="%{WORD:severity}" sys="%{WORD:sys}" sub="%{WORD:sub}" name="%{DATA:name}" action="%{WORD:action}" fwrule="%{INT:fwrule}" initf="%{NOTSPACE:initf}" srcmac="%{MAC:srcmac}" dstmac="%{MAC:dstmac}" srcip="%{IPV4:source_ip}" dstip="%{IPV4:destination_ip}" proto="%{WORD:protocol}" length="%{INT:length}" tos="%{DATA:tos}" prec="%{DATA:prec}" ttl="%{INT:ttl}" srcport="%{INT:srcport}" dstport="%{INT:dstport}" "]
}
mutate {
remove_tag => "_grokparsefailure"
add_tag => "ulogd" }
geoip {
source => "destination_ip"
target => "geoip"
database => "/etc/logstash/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
...
}