Grokparsefailure! wtf?

Elastic Stack Logstash
1

Hi there,

After using grokdebug and make sure the grok filter as working, just find out that was not enough to avoid the tag _grokparsefailure!! arggggggg!!!

Follow my conf:

###################################### INPUT ###########################################################

input {
 udp { 
 port => 5000
 type => "cisco-fw"
 }
}

############################################ BEGIN FILTER ###############################################

filter {
 if [type] == "cisco-fw" {
 grok {
 named_captures_only => true
 match => ["message", "%{CISCO_TAGGED_SYSLOG} %{GREEDYDATA:cisco_message}"]
 }
 # Parse the syslog severity and facility
 syslog_pri { }


 grok {
 patterns_dir => "/etc/logstash/conf.d/patterns/"
 match => [
 "cisco_message", "%{CISCOFW106001}",
 "cisco_message", "%{CISCOFW106006_106007_106010}",
 "cisco_message", "%{CISCOFW106014}",
 "cisco_message", "%{CISCOFW106015}",
 "cisco_message", "%{CISCOFW106021}",
 "cisco_message", "%{CISCOFW106023}",
 "cisco_message", "%{CISCOFW106100}",
 "cisco_message", "%{CISCOFW110002}",
 "cisco_message", "%{CISCOFW302010}",
 "cisco_message", "%{CISCOFW302013_302014_302015_302016}",
 "cisco_message", "%{CISCOFW302020_302021}",
 "cisco_message", "%{CISCOFW305011}",
 "cisco_message", "%{CISCOFW313001_313004_313008}",
 "cisco_message", "%{CISCOFW313005}",
 "cisco_message", "%{CISCOFW402117}",
 "cisco_message", "%{CISCOFW402119}",
 "cisco_message", "%{CISCOFW419001}",
 "cisco_message", "%{CISCOFW419002}",
 "cisco_message", "%{CISCOFW500004}",
 "cisco_message", "%{CISCOFW602303_602304}",
 "cisco_message", "%{CISCOFW710001_710002_710003_710005_710006}",
 "cisco_message", "%{CISCOFW713172}",
 "cisco_message", "%{CISCOFW733100}"
 ]
 }
 
 #If CiscoTag is ASA-4-722051 - VPN Assigned to a session
 if [type] == "cisco-fw" and [ciscotag] == "ASA-4-722051" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> IPv4 Address <%{IPV4:assigned_ip}%{GREEDYDATA:extra_field}"]
 }
}

  #If CiscoTag is ASA-6-722055 - Cisco VPN agent being used to establish the connection
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722055" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}"]
 }
}

 #If CiscoTag is ASA-6-113005 - VPN Authentication Rejected
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-113005" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s%{GREEDYDATA:Policy} : reason =\s%{DATA:Reason} : server =\s%{IPV4:ServerIP} : user =\s%{DATA:[event_data][TargetUserName]} : user IP =\s%{IPV4:src_ip}"]
 }
}
 
 #If CiscoTag is ASA-6-722023 - VPN Connection Terminated
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722023" {
 grok {
 match => ["cisco_message", "<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> %{GREEDYDATA:action}"]
 }
}
 
 #If CiscoTag is ASA-4-113019 - VPN user logout
 if [type] == "cisco-fw" and [ciscotag] == "ASA-4-113019" {
 grok {
 match => ["cisco_message", "Group = %{DATA:group}, Username = %{DATA:user}, IP = %{IP:src_ip}, %{DATA:action}\. Session Type: %{DATA:session_type}, Duration: %{DATA:duration}, Bytes xmt: %{INT:bytes_xmt}, Bytes rcv: %{INT:bytes_rcv}, Reason: %{GREEDYDATA:reason}"]
 }
}
 
 #If CiscoTag is ASA-6-113039 - VPN user successfully logged in
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-113039" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> %{GREEDYDATA:action}"]
 }
}
 
 

if [type] == "cisco-fw" {
 geoip {
 add_tag => [ "GeoIP" ]
 database => "/etc/logstash/GeoLite2-City.mmdb" ### Change me to location of GeoLiteCity.dat file
 source => "src_ip"
 }

 if [geoip][city_name] == "" { mutate { remove_field => "[geoip][city_name]" } }
 if [geoip][continent_code] == "" { mutate { remove_field => "[geoip][continent_code]" } }
 if [geoip][country_code2] == "" { mutate { remove_field => "[geoip][country_code2]" } }
 if [geoip][country_code3] == "" { mutate { remove_field => "[geoip][country_code3]" } }
 if [geoip][country_name] == "" { mutate { remove_field => "[geoip][country_name]" } }
 if [geoip][latitude] == "" { mutate { remove_field => "[geoip][latitude]" } }
 if [geoip][longitude] == "" { mutate { remove_field => "[geoip][longitude]" } }
 if [geoip][postal_code] == "" { mutate { remove_field => "[geoip][postal_code]" } }
 if [geoip][region_name] == "" { mutate { remove_field => "[geoip][region_name]" } }
 if [geoip][time_zone] == "" { mutate { remove_field => "[geoip][time_zone]" } }
 }


 # Parse the date
 date {
 match => ["timestamp",
 "MMM dd HH:mm:ss",
 "MMM d HH:mm:ss",
 "MMM dd yyyy HH:mm:ss",
 "MMM d yyyy HH:mm:ss"
 ]
 }
# if "_grokparsefailure" in [tags] {
#  drop { }
#}

 }
}

########################################### END FILTER ##################################################

output {
if [type] == "cisco-fw" and ([ciscotag] == "ASA-4-722051" or [ciscotag] == "ASA-6-722055" or [ciscotag] == "ASA-6-113005" or [ciscotag] == "ASA-6-722023" or [ciscotag] == "ASA-4-113019" or [ciscotag] == "ASA-6-113039") {
    elasticsearch {
    hosts => ["http://localhost:9200"]
    user => "${ES_USER}"
    password => "${ES_PWD}"
    index => "logstash-ciscovpn-%{+yyyy.MM.dd}"

  }
 }
}

And the result using this grokfilter:

%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}

"message" => "<166>May 15 2020 16:08:16: %ASA-6-722055: Group <GroupPolicy_Resource_Remote> User <vpn.test> IP <xxx.xxx.xx.xxx> Client Type: Cisco AnyConnect VPN Agent for Windows 4.8.03036\n",
           "cisco_message" => "Group <GroupPolicy_Resource_Remote> User <vpn.test> IP <xxx.xxx.xx.xxx> Client Type: Cisco AnyConnect VPN Agent for Windows 4.8.03036\n",
              "@timestamp" => 2020-05-15T19:08:16.000Z,
               "timestamp" => "May 15 2020 16:08:16",
                    "host" => "192.168.1.2",
         "syslog_severity" => "informational",
         "syslog_facility" => "local4",
                "ciscotag" => "ASA-6-722055",
                "@version" => "1",
                  "Policy" => "GroupPolicy_Resource_Remote",
                  "src_ip" => "xxx.xxx.xx.xxx",
              "syslog_pri" => "166",
                    "tags" => [
        [0] "_grokparsefailure",
        [1] "GeoIP"
    ],
    "syslog_facility_code" => 20,
                   "Group" => "Group",
    "syslog_severity_code" => 6,
              "event_data" => {
        "TargetUserName" => "vpn.test"
    },
              "VPN_Client" => " Cisco AnyConnect VPN Agent for Windows 4.8.03036\n",
                    "type" => "cisco-fw"
}
{
                   "geoip" => {
        "continent_code" => "SA",
                    "ip" => "xxx.xxx.xx.xxx",
         "country_code2" => "BR",
           "postal_code" => "01000",
             "longitude" => -46.6322,
           "region_name" => "Sao Paulo",
          "country_name" => "Brazil",
         "country_code3" => "BR",
              "location" => {
            "lon" => -46.6322,
            "lat" => -23.63
        },
           "region_code" => "SP",
              "latitude" => -23.63,
             "city_name" => "São Paulo",
              "timezone" => "America/Sao_Paulo"
    },

Any light at the end of the tunnel??? Can someone help me to understand why this happening?

Thanks all for the attention.

2

I suggest you add tag_on_failure options to your grok filters to isolate which filter is failing.

3

@Badger do you mean adding in all of the grok filter?? Or one by one? Like this:

if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722055" {
 grok {
 break_on_match => true
 tag_on_failure => ["parse-failed"]
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}"]
 }
}
4

I mean add a different tag in every grok filter.

5

@Badger could you give an example using my conf? Just to make sure I'm on the right path?

6

I mean you modify all of your grok filters like this:

#If CiscoTag is ASA-4-722051 - VPN Assigned to a session
if [type] == "cisco-fw" and [ciscotag] == "ASA-4-722051" {
    grok {
        match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> IPv4 Address <%{IPV4:assigned_ip}%{GREEDYDATA:extra_field}"]
        tag_on_failure => ["_grokparsefailure", "grok722051failure"]
    }
}
#If CiscoTag is ASA-6-722055 - Cisco VPN agent being used to establish the connection
if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722055" {
    grok {
        match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}"]
        tag_on_failure => ["_grokparsefailure", "grok722055failure"]
    }
}

Do that for every grok using a unique value and you will then know which grok is failing.

7

@Badger done that but unfortunately is not working. I can´t any tag like eg.
grok722055failure
I should be happy or not? The _grokparsefailure still apears.

8

@Badger Found the solution!!! The problem was because I don´t use pipelines and templates, logstash template automatically adds new fields. As soon as I deleted the index and recreated, the _grokparsefailure gone!

Definitely I need to find out how to work with pipelines and templates.

Thanks one more time for your attention. Really appreciate