Grokparsefailure! wtf?

Hi there,

After using grokdebug and make sure the grok filter as working, just find out that was not enough to avoid the tag _grokparsefailure!! arggggggg!!!

Follow my conf:

###################################### INPUT ###########################################################

input {
 udp { 
 port => 5000
 type => "cisco-fw"
 }
}

############################################ BEGIN FILTER ###############################################

filter {
 if [type] == "cisco-fw" {
 grok {
 named_captures_only => true
 match => ["message", "%{CISCO_TAGGED_SYSLOG} %{GREEDYDATA:cisco_message}"]
 }
 # Parse the syslog severity and facility
 syslog_pri { }


 grok {
 patterns_dir => "/etc/logstash/conf.d/patterns/"
 match => [
 "cisco_message", "%{CISCOFW106001}",
 "cisco_message", "%{CISCOFW106006_106007_106010}",
 "cisco_message", "%{CISCOFW106014}",
 "cisco_message", "%{CISCOFW106015}",
 "cisco_message", "%{CISCOFW106021}",
 "cisco_message", "%{CISCOFW106023}",
 "cisco_message", "%{CISCOFW106100}",
 "cisco_message", "%{CISCOFW110002}",
 "cisco_message", "%{CISCOFW302010}",
 "cisco_message", "%{CISCOFW302013_302014_302015_302016}",
 "cisco_message", "%{CISCOFW302020_302021}",
 "cisco_message", "%{CISCOFW305011}",
 "cisco_message", "%{CISCOFW313001_313004_313008}",
 "cisco_message", "%{CISCOFW313005}",
 "cisco_message", "%{CISCOFW402117}",
 "cisco_message", "%{CISCOFW402119}",
 "cisco_message", "%{CISCOFW419001}",
 "cisco_message", "%{CISCOFW419002}",
 "cisco_message", "%{CISCOFW500004}",
 "cisco_message", "%{CISCOFW602303_602304}",
 "cisco_message", "%{CISCOFW710001_710002_710003_710005_710006}",
 "cisco_message", "%{CISCOFW713172}",
 "cisco_message", "%{CISCOFW733100}"
 ]
 }
 
 #If CiscoTag is ASA-4-722051 - VPN Assigned to a session
 if [type] == "cisco-fw" and [ciscotag] == "ASA-4-722051" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> IPv4 Address <%{IPV4:assigned_ip}%{GREEDYDATA:extra_field}"]
 }
}

  #If CiscoTag is ASA-6-722055 - Cisco VPN agent being used to establish the connection
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722055" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}"]
 }
}

 #If CiscoTag is ASA-6-113005 - VPN Authentication Rejected
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-113005" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s%{GREEDYDATA:Policy} : reason =\s%{DATA:Reason} : server =\s%{IPV4:ServerIP} : user =\s%{DATA:[event_data][TargetUserName]} : user IP =\s%{IPV4:src_ip}"]
 }
}
 
 #If CiscoTag is ASA-6-722023 - VPN Connection Terminated
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722023" {
 grok {
 match => ["cisco_message", "<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> %{GREEDYDATA:action}"]
 }
}
 
 #If CiscoTag is ASA-4-113019 - VPN user logout
 if [type] == "cisco-fw" and [ciscotag] == "ASA-4-113019" {
 grok {
 match => ["cisco_message", "Group = %{DATA:group}, Username = %{DATA:user}, IP = %{IP:src_ip}, %{DATA:action}\. Session Type: %{DATA:session_type}, Duration: %{DATA:duration}, Bytes xmt: %{INT:bytes_xmt}, Bytes rcv: %{INT:bytes_rcv}, Reason: %{GREEDYDATA:reason}"]
 }
}
 
 #If CiscoTag is ASA-6-113039 - VPN user successfully logged in
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-113039" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> %{GREEDYDATA:action}"]
 }
}
 
 

if [type] == "cisco-fw" {
 geoip {
 add_tag => [ "GeoIP" ]
 database => "/etc/logstash/GeoLite2-City.mmdb" ### Change me to location of GeoLiteCity.dat file
 source => "src_ip"
 }

 if [geoip][city_name] == "" { mutate { remove_field => "[geoip][city_name]" } }
 if [geoip][continent_code] == "" { mutate { remove_field => "[geoip][continent_code]" } }
 if [geoip][country_code2] == "" { mutate { remove_field => "[geoip][country_code2]" } }
 if [geoip][country_code3] == "" { mutate { remove_field => "[geoip][country_code3]" } }
 if [geoip][country_name] == "" { mutate { remove_field => "[geoip][country_name]" } }
 if [geoip][latitude] == "" { mutate { remove_field => "[geoip][latitude]" } }
 if [geoip][longitude] == "" { mutate { remove_field => "[geoip][longitude]" } }
 if [geoip][postal_code] == "" { mutate { remove_field => "[geoip][postal_code]" } }
 if [geoip][region_name] == "" { mutate { remove_field => "[geoip][region_name]" } }
 if [geoip][time_zone] == "" { mutate { remove_field => "[geoip][time_zone]" } }
 }


 # Parse the date
 date {
 match => ["timestamp",
 "MMM dd HH:mm:ss",
 "MMM d HH:mm:ss",
 "MMM dd yyyy HH:mm:ss",
 "MMM d yyyy HH:mm:ss"
 ]
 }
# if "_grokparsefailure" in [tags] {
#  drop { }
#}

 }
}

########################################### END FILTER ##################################################

output {
if [type] == "cisco-fw" and ([ciscotag] == "ASA-4-722051" or [ciscotag] == "ASA-6-722055" or [ciscotag] == "ASA-6-113005" or [ciscotag] == "ASA-6-722023" or [ciscotag] == "ASA-4-113019" or [ciscotag] == "ASA-6-113039") {
    elasticsearch {
    hosts => ["http://localhost:9200"]
    user => "${ES_USER}"
    password => "${ES_PWD}"
    index => "logstash-ciscovpn-%{+yyyy.MM.dd}"

  }
 }
}

And the result using this grokfilter:

%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}
"message" => "<166>May 15 2020 16:08:16: %ASA-6-722055: Group <GroupPolicy_Resource_Remote> User <vpn.test> IP <xxx.xxx.xx.xxx> Client Type: Cisco AnyConnect VPN Agent for Windows 4.8.03036\n",
           "cisco_message" => "Group <GroupPolicy_Resource_Remote> User <vpn.test> IP <xxx.xxx.xx.xxx> Client Type: Cisco AnyConnect VPN Agent for Windows 4.8.03036\n",
              "@timestamp" => 2020-05-15T19:08:16.000Z,
               "timestamp" => "May 15 2020 16:08:16",
                    "host" => "192.168.1.2",
         "syslog_severity" => "informational",
         "syslog_facility" => "local4",
                "ciscotag" => "ASA-6-722055",
                "@version" => "1",
                  "Policy" => "GroupPolicy_Resource_Remote",
                  "src_ip" => "xxx.xxx.xx.xxx",
              "syslog_pri" => "166",
                    "tags" => [
        [0] "_grokparsefailure",
        [1] "GeoIP"
    ],
    "syslog_facility_code" => 20,
                   "Group" => "Group",
    "syslog_severity_code" => 6,
              "event_data" => {
        "TargetUserName" => "vpn.test"
    },
              "VPN_Client" => " Cisco AnyConnect VPN Agent for Windows 4.8.03036\n",
                    "type" => "cisco-fw"
}
{
                   "geoip" => {
        "continent_code" => "SA",
                    "ip" => "xxx.xxx.xx.xxx",
         "country_code2" => "BR",
           "postal_code" => "01000",
             "longitude" => -46.6322,
           "region_name" => "Sao Paulo",
          "country_name" => "Brazil",
         "country_code3" => "BR",
              "location" => {
            "lon" => -46.6322,
            "lat" => -23.63
        },
           "region_code" => "SP",
              "latitude" => -23.63,
             "city_name" => "São Paulo",
              "timezone" => "America/Sao_Paulo"
    },

Any light at the end of the tunnel??? Can someone help me to understand why this happening?

Thanks all for the attention.

I suggest you add tag_on_failure options to your grok filters to isolate which filter is failing.

@Badger do you mean adding in all of the grok filter?? Or one by one? Like this:

if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722055" {
 grok {
 break_on_match => true
 tag_on_failure => ["parse-failed"]
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}"]
 }
}

I mean add a different tag in every grok filter.

@Badger could you give an example using my conf? Just to make sure I'm on the right path?

I mean you modify all of your grok filters like this:

#If CiscoTag is ASA-4-722051 - VPN Assigned to a session
if [type] == "cisco-fw" and [ciscotag] == "ASA-4-722051" {
    grok {
        match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> IPv4 Address <%{IPV4:assigned_ip}%{GREEDYDATA:extra_field}"]
        tag_on_failure => ["_grokparsefailure", "grok722051failure"]
    }
}
#If CiscoTag is ASA-6-722055 - Cisco VPN agent being used to establish the connection
if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722055" {
    grok {
        match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}"]
        tag_on_failure => ["_grokparsefailure", "grok722055failure"]
    }
}

Do that for every grok using a unique value and you will then know which grok is failing.

@Badger done that but unfortunately is not working. I can´t any tag like eg.
grok722055failure
I should be happy or not? The _grokparsefailure still apears.

@Badger Found the solution!!! The problem was because I don´t use pipelines and templates, logstash template automatically adds new fields. As soon as I deleted the index and recreated, the _grokparsefailure gone!

Definitely I need to find out how to work with pipelines and templates.

Thanks one more time for your attention. Really appreciate

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.