Hello community,
I have been encountering for some time a problem with the use of 100% of my CPU, on my logstash configuration.
The problem occurred overnight.
I am using the docker version of logstash version 7.5.2.
Below my configuration.
input { syslog {
port => 5044
}
}
# input {
# beats {
# port => 5043
# }
# }
filter {
if [logsource] == "192.168.10.254" {
mutate { add_tag => "pfsense" }
}
}
filter {
if "pfsense" in [tags] and [program] == "filterlog" {
grok {
match => { "message" => ".*?,.*?,.*?,%{NUMBER:ID_RULE},%{WORD:INTERFACE},.*?,%{WORD:ACTION},%{WORD:TRAFIC},%{NUMBER:IP_VERSION},.*?,.*?,.*?,.*?,.*?,.*?,.*?,%{WORD:PROTOCOL},.*?,%{IP:IP_SOURCE},%{IP:IP_DESTINATION},%{NUMBER:PORT_SOURCE},%{NUMBER:PORT_DESTINATION},%{NUMBER:DATA_LENGHT}" }
match => { "message" => ".*?,.*?,.*?,%{NUMBER:ID_RULE},%{WORD:INTERFACE},.*?,%{WORD:ACTION},%{WORD:TRAFIC},%{NUMBER:IP_VERSION},.*?,*?,.*?,.*?,%{WORD:PROTOCOL},.*?,%{IP:IP_SOURCE},%{IP:IP_DESTINATION},%{NUMBER:PORT_SOURCE},%{NUMBER:PORT_DESTINATION},%{NUMBER:DATA_LENGHT}" }
add_tag => "pfsense-filterlog"
}
geoip {
source => "IP_SOURCE"
target => "srcgeoip"
}
geoip {
source => "IP_DESTINATION"
target => "dstgeoip"
}
}
if "pfsense" in [tags] and [program] == "pfblockerng_ip" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP},%{NUMBER:ID_RULE},%{WORD:INTERFACE},%{WORD:INTERFACE_NAME},%{WORD:ACTION},%{NUMBER:IP_VERSION},.*?,%{WORD:PROTOCOL}.*?,%{IP:IP_SOURCE},%{IP:IP_DESTINATION},%{NUMBER:PORT_SOURCE},%{NUMBER:PORT_DESTINATION},%{WORD:TRAFIC},%{WORD:GEOIP},%{WORD:ALIAS_GEO},.*?,%{WORD:ALIAS_NAME}," }
add_tag => "pfsense-pfblockerng_ip"
}
geoip {
source => "IP_SOURCE"
target => "srcgeoip"
}
geoip {
source => "IP_DESTINATION"
target => "dstgeoip"
}
geoip {
source => "IP_EVALUATED"
target => "srcgeoip"
}
}
if "pfsense" in [tags] and [program] == "barnyard2" {
grok {
patterns_dir => ["/usr/share/logstash/pipeline/patterns/pfsense"]
match => { "message" => "\[%{SID:SID}]\s%{DESCRIPTION:DESCRIPTION}\s\[Classification\:\s%{CLASSIFICATION:CLASSIFICATION}\]\s\[Priority:\s%{NUMBER:PRIORITY}\]\:\s\<%{WORD:INTERFACE}\>\s\{%{WORD:PROTOCOL}\}\s%{IP:IP_SOURCE}\:%{NUMBER:PORT_SOURCE}\s\-\>\s%{IP:IP_DESTINATION}\:%{NUMBER:PORT_DESTINATION}" }
add_tag => "pfsense-barnyard2"
}
geoip {
source => "IP_SOURCE"
target => "srcgeoip"
}
geoip {
source => "IP_DESTINATION"
target => "dstgeoip"
}
}
if "pfsense" in [tags] and [program] == "unbound" {
grok {
patterns_dir => ["/usr/share/logstash/pipeline/patterns/pfsense"]
break_on_match => false
match => { "message" => [
".*?info:\s%{IP:IP_SOURCE}\s%{FQDN:FQDN}\.\s",
".*?info:\s.*?\s.*?%{DOMAIN:DOMAIN}\.\s",
".*?info:\s.*?\s.*?%{TLD:TLD}\.\s" ] }
add_tag => "pfsense-unbound"
}
}
if "pfsense" in [tags] and [program] == "pfblockerng_dnsbl" {
grok {
patterns_dir => ["/usr/share/logstash/pipeline/patterns/pfsense"]
break_on_match => false
match => { "message" => [
".*?%{SYSLOGTIMESTAMP},%{HOSTNAME:FQDN},%{IP:IP_SOURCE},.*?,.*?,%{WORD:ALIASGROUP},.*?,%{WORD:ALIASNAME},",
".*?%{SYSLOGTIMESTAMP},.*?%{DOMAIN:DOMAIN },",
".*?%{SYSLOGTIMESTAMP},.*?%{TLD:TLD}," ] }
add_tag => "pfsense-pfblockerng_dnsbl"
}
}
}
output {
if "pfsense-filterlog" in [tags] {
elasticsearch {
hosts => "10.0.7.2:9200"
index => "pfsense-filterlog-%{+YYYY.MM.dd}"
}
}
if "pfsense-pfblockerng_ip" in [tags] {
elasticsearch {
hosts => "10.0.7.2:9200"
index => "pfsense-pfblockerng_ip-%{+YYYY.MM.dd}"
}
}
if "pfsense-barnyard2" in [tags] {
elasticsearch {
hosts => "10.0.7.2:9200"
index => "pfsense-barnyard2-%{+YYYY.MM.dd}"
}
}
if "pfsense-unbound" in [tags] {
elasticsearch {
hosts => "10.0.7.2:9200"
index => "pfsense-unbound-%{+YYYY.MM.dd}"
}
}
if "pfsense-pfblockerng_dnsbl" in [tags] {
elasticsearch {
hosts => "10.0.7.2:9200"
index => "pfsense-pfblockerng_dnsbl-%{+YYYY.MM.dd}"
}
}
if "netflow" in [tags] {
elasticsearch {
hosts => "10.0.7.2:9200"
index => "netflow-%{+YYYY.MM.dd}"
}
}
else {
elasticsearch {
hosts => "10.0.7.2:9200"
index => "logstash-%{+YYYY.MM.dd}"
}
}
}
Thank you.