Grok is succesfull in various debuggers but fails in practice with specific keyword

Elastic Stack Logstash
1

Hi, I have the following logstash config which parses syslog messages:

filter {
  if "syslog" in [tags] and "pre-processed" not in [tags] {
    if "%ASA-" in [message] {
      mutate {
        gsub => [
            "message", "<166>", "",
            "message", "<164>", "",
            "message", "<163>", "",
            "message", "<162>", ""
        ]
        add_tag => [ "pre-processed", "Firewall", "ASA", "log01" ]
      }
      grok {
        match => [
             "message", "^%{CISCOTIMESTAMP:localtime}:.%%{CISCOTAG:ciscotag}:.%{CISCO_ACTION:action}.%{WORD}.%{WORD}.%{WORD}.%{WORD}.%{WORD}:%{IPORHOST}/%{NUMBER}.%{WORD}.%{WORD}:%{IPORHOST:dst_ip}/%{NUMBER:dst_port}$",
             "message", "^%{CISCOTIMESTAMP:localtime}.%{IPORHOST:host}.%%{CISCOTAG:ciscotag}: SFR requested ASA to %{WORD:action} further packet redirection and process %{WORD:protocol} flow from %{IPORHOST:src_interface}:%{IPORHOST:src_ip}/%{NUMBER:src_port} to %{IPORHOST:dst_interface}:%{IPORHOST:dst_ip}/%{NUMBER:dst_port} %{GREEDYDATA:cisco_message}$",
             "message", "^%{CISCOTIMESTAMP:localtime}.%{IPORHOST:host}.%%{CISCOTAG:ciscotag}: SFR requested to %{WORD:action} %{WORD:protocol} packet from %{IPORHOST:src_interface}:%{IPORHOST:src_ip}/%{NUMBER:src_port}$",
             "message", "^%{CISCOTIMESTAMP:localtime}:.%%{CISCOTAG:ciscotag}:.%{GREEDYDATA:cisco_message}$"
       ]
      }
      syslog_pri { }

        if "_grokparsefailure" not in [tags] {
          mutate {
          rename => ["cisco_message", "message"]
          remove_field => ["timestamp"]
          }
        }

 grok {
      match => [
        "message", "%{CISCOFW106001}",
        "message", "%{CISCOFW106006_106007_106010}",
        "message", "%{CISCOFW106014}",
        "message", "%{CISCOFW106015}",
        "message", "%{CISCOFW106021}",
        "message", "%{CISCOFW106023}",
        "message", "%{CISCOFW106100}",
        "message", "%{CISCOFW110002}",
        "message", "%{CISCOFW302010}",
        "message", "%{CISCOFW302013_302014_302015_302016}",
        "message", "%{CISCOFW302020_302021}",
        "message", "%{CISCOFW305011}",
        "message", "%{CISCOFW313001_313004_313008}",
        "message", "%{CISCOFW313005}",
        "message", "%{CISCOFW402117}",
        "message", "%{CISCOFW402119}",
        "message", "%{CISCOFW419001}",
        "message", "%{CISCOFW419002}",
        "message", "%{CISCOFW500004}",
        "message", "%{CISCOFW602303_602304}",
        "message", "%{CISCOFW710001_710002_710003_710005_710006}",
        "message", "%{CISCOFW713172}",
        "message", "%{CISCOFW733100}"
      ]
    }
   geoip {
        source => "src_ip"
        target => "geo_point"
       }
    }
  }
}

And i have the following logmessages:

Jul 16 10:30:45 123.123.123.123 %ASA-4-434002: SFR requested to drop TCP packet from OUTSIDE-VRF180:123.123.123.123/80 to INSIDE-VRF4100:123.123.123.123/2651

According to my setup, the log message above should match the third regex, which is the following:

^%{CISCOTIMESTAMP:localtime}.%{IPORHOST:host}.%%{CISCOTAG:ciscotag}: SFR requested to %{WORD:action} %{WORD:protocol} packet from %{IPORHOST:src_interface}:%{IPORHOST:src_ip}/%{NUMBER:src_port}$

But when i view it in kibana, i can see that it gets tagged "_grokparsefailure" and lacks any keyword after ciscotag, i.e. "action", "protocol" etc is missing while "ciscotag", "localtime" etc works fine.
I have tried the grok expresseion in various online debuggers as well as the built-in debugger in kibana which shows that there are no errors and should work.

Can anyone help identify where the problem lies?

2

The first part shoudl match. I would not expect

%{IPORHOST:src_interface}:%{IPORHOST:src_ip}/%{NUMBER:src_port}$

(anchored to end of line) to match

OUTSIDE-VRF180:123.123.123.123/80 to INSIDE-VRF4100:123.123.123.123/2651
3

Ah so sorry, i pasted the wrong grok expression for the log i submitted. That's fixed now. You are indeed correct in that it doesn't work. I think i've fixed it with the following regex:

%{CISCOTIMESTAMP:localtime}.%{IPORHOST:host} %%{CISCOTAG:ciscotag}: %{WORD} %{WORD} %{WORD} %{WORD:action} %{WORD:protocol} %{WORD} %{WORD} %{HOSTNAME:src_interface}:%{IP:src_ip}/%{NUMBER:src_port} %{WORD} %{HOSTNAME:dst_interface}:%{IP:dst_ip}/%{NUMBER:dst_port}

EDIT: Definetly not fixed, still has the same issues as before.

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.