I'm new to logstash, and writing a config to parse logs from the firewall appliance in my home office. I've debugged a few issues with this config already and now I have it to a place where it works without any errors. However, there are fields I am trying to populate that rely on some conditionals that just don't ever seem to be matched.
Specifically, the mutates to add event.type, event.action, and the conditional grok statements to break up the 'transport options' field depending on whether the protocol is tcp, udp, or ICMP are not working as expected. Am I using these conditionals incorrectly?
Filter statement:
filter {
grok {
match => { "message" => "%{NONNEGINT:facility}>%{SYSLOGTIMESTAMP:@timestamp} %{HOSTNAME:host.name} %{WORD:event.provider}: \[%{GREEDYDATA:rule.name}\]IN=%{GREEDYDATA:interface.in} OUT=%{GREEDYDATA:interface.out} MAC=%{MAC:destination.mac}\:%{MAC:source.mac}\:%{GREEDYDATA:ethertype} SRC=%{IP:source.ip} DST=%{IP:destination.ip} LEN=%{INT:destination.bytes} TOS=%{GREEDYDATA} PREC=%{GREEDYDATA} TTL=%{INT:network.ttl} ID=%{INT:network.packet.id}\s(CE\s|DF\s|MF\s)?PROTO=%{WORD:network.transport} %{GREEDYDATA:transport.options}"
}
}
mutate {
remove_field => [ "host" ]
add_field => { "observer.hostname" => "%{host.name}" }
add_field => { "observer.type" => "firewall" }
add_field => { "event.category" => "network" }
add_field => { "event.type" => "connection" }
}
if [rule][name] =~ /A$/ {
mutate {
add_field => { "event.type" => "allowed" }
add_field => { "event.action" => "accept" }
}
}
if [rule][name] =~ /D$/ {
mutate {
add_field => { "event.type" => "denied" }
add_field => { "event.action" => "drop" }
}
}
if [rule][name] =~ /R$/ {
mutate {
add_field => { "event.type" => "denied" }
add_field => { "event.action" => "reject" }
}
}
if [network][transport] == "TCP"{
grok {
match => [ "transport.options", "SPT=%{INT:source.port} DPT=%{INT:destination.port} WINDOW=%{INT:network.tcp.window} RES=(?<network.tcp.res>[0-9]x[0-9][0-9]) %{GREEDYDATA:network.tcp.flags}"]
}
mutate {
split => { "network.tcp.flags" => " " }
remove_field => "transport.options"
}
}
if [network][transport] == "UDP"{
grok {
match => [ "transport.options", "SPT=%{INT:source.port} DPT=%{INT:destination.port} LEN=%{INT:network.udp.bytes}"]
}
mutate {
remove_field => "transport.options"
}
}
if [network][transport] == "ICMP"{
grok {
match => [ "transport.options", "TYPE=%{INT:network.icmp.type} CODE=%{INT:network.icmp.code} ID=%{INT:network.icmp.id} SEQ=%{INT:network.icmp.sequence}"]
}
mutate {
remove_field => "transport.options"
}
}
if "WAN" in [rule][name] {
geoip { source => "source.ip" }
}
if "LAN" in [rule][name] {
geoip { source => "destination.ip" }
}
}
Sample logs:
<4>Nov 28 17:30:41 USG-1 kernel: [WAN_LOCAL-default-D]IN=eth0 OUT= MAC=e0:63:da:22:b3:c2:00:01:5c:a7:aa:45:08:00 SRC=196.64.21.2 DST=96.51.17.173 LEN=44 TOS=0x00 PREC=0x00 TTL=236 ID=44963 PROTO=TCP SPT=47676 DPT=1433 WINDOW=1024 RES=0x00 SYN URGP=0
<4>Nov 28 17:31:31 USG-1 kernel: [WAN_LOCAL-default-D]IN=eth0 OUT= MAC=e0:63:da:22:b3:c2:00:01:5c:a7:aa:45:08:00 SRC=146.88.240.4 DST=96.51.17.173 LEN=29 TOS=0x00 PREC=0x00 TTL=250 ID=54321 PROTO=UDP SPT=37540 DPT=19 LEN=9
<4>Nov 28 17:30:36 USG-1 kernel: [LAN_IN-4000-A]IN=eth1 OUT=eth0 MAC=e0:63:da:22:b3:c3:2c:fd:a1:bc:0f:65:08:00 SRC=192.168.1.130 DST=172.217.3.206 LEN=1378 TOS=0x00 PREC=0x00 TTL=127 ID=4251 DF PROTO=UDP SPT=59708 DPT=443 LEN=1358
<4>Nov 28 17:30:36 USG-1 kernel: [LAN_IN-4000-A]IN=eth1 OUT=eth0 MAC=e0:63:da:22:b3:c3:2c:fd:a1:bc:0f:65:08:00 SRC=192.168.1.130 DST=172.217.3.206 LEN=52 TOS=0x00 PREC=0x00 TTL=127 ID=4252 DF PROTO=TCP SPT=49187 DPT=443 WINDOW=64240 RES=0x00 SYN URGP=0
<4>Nov 28 17:30:36 USG-1 kernel: [LAN_IN-4000-A]IN=eth1 OUT=eth0 MAC=e0:63:da:22:b3:c3:2c:fd:a1:bc:0f:65:08:00 SRC=192.168.1.130 DST=8.8.8.8 LEN=60 TOS=0x00 PREC=0x00 TTL=127 ID=16630 PROTO=ICMP TYPE=8 CODE=0 ID=1 SEQ=965
stdout on the logstash container:
{
"source.mac" => "1c:87:2c:43:05:e5",
"network.packet.id" => "64827",
"network.transport" => "TCP",
"@version" => "1",
"network.ttl" => "127",
"event.category" => "network",
"source.ip" => "192.168.1.134",
"ethertype" => "08:00",
"observer.type" => "firewall",
"interface.in" => "eth1",
"type" => "syslog",
"observer.hostname" => "USG-1",
"host.name" => "USG-1",
"rule.name" => "LAN_IN-4000-A",
"interface.out" => "eth0",
"event.type" => "connection",
"destination.ip" => "216.58.217.37",
"@timestamp" => 2020-11-29T18:48:28.743Z,
"destination.bytes" => "41",
"event.provider" => "kernel",
"facility" => "4",
"destination.mac" => "e0:63:da:22:b3:c3",
"transport.options" => "SPT=55950 DPT=443 WINDOW=1025 RES=0x00 ACK URGP=0 ",
"message" => "<4>Nov 29 11:48:28 USG-1 kernel: [LAN_IN-4000-A]IN=eth1 OUT=eth0 MAC=e0:63:da:22:b3:c3:1c:87:2c:43:05:e5:08:00 SRC=192.168.1.134 DST=216.58.217.37 LEN=41 TOS=0x00 PREC=0x00 TTL=127 ID=64827 DF PROTO=TCP SPT=55950 DPT=443 WINDOW=1025 RES=0x00 ACK URGP=0 "
}