CISCO Grok Fail

Let me say I am a network guy so take that for what it is worth. I have spent weeks googling, testing, and debugging I tried not to hassle anyone.

I am trying to Grok the following data.

<134>May 08 2020 10:50:53: %ASA-6-734001: DAP: User xxxx,xxxx, Addr xx.xx.xx.xx, Connection AnyConnect: The following DAP records were selected for this connection: XXXX-XXX-XXX```

Here is my Grok filter

#if CiscoTAG is ASA-6-734001
        if [type] == "cisco-fw" and [ciscotag] == "%ASA-6-734001" {
        grok {
        match => ["cisco_message", "DAP: User %{DATA:user}, Addr %{IP:src_ip}, Connection %{DATA:protocol}: The following DAP records were selected for this connection: %{GREEDYDATA:policy_id1}"]
         }
        }

Here is my config file

input { stdin {} }

filter {
        grok {
                match => ["message", "%{CISCO_TAGGED_SYSLOG} %{GREEDYDATA:cisco_message}"]
        }

        # Extract fields from the each of the detailed message types
        # The patterns provided below are included in core of LogStash 1.4.2.
        grok {
                match => [
                        "cisco_message", "%{CISCOFW106001}",
                        "cisco_message", "%{CISCOFW106006_106007_106010}",
                        "cisco_message", "%{CISCOFW106014}",
                        "cisco_message", "%{CISCOFW106015}",
                        "cisco_message", "%{CISCOFW106021}",
                        "cisco_message", "%{CISCOFW106023}",
                        "cisco_message", "%{CISCOFW106100}",
                        "cisco_message", "%{CISCOFW110002}",
                        "cisco_message", "%{CISCOFW302010}",
                        "cisco_message", "%{CISCOFW302013_302014_302015_302016}",
                        "cisco_message", "%{CISCOFW302020_302021}",
                        "cisco_message", "%{CISCOFW305011}",
                        "cisco_message", "%{CISCOFW313001_313004_313008}",
                        "cisco_message", "%{CISCOFW313005}",
                        "cisco_message", "%{CISCOFW402117}",
                        "cisco_message", "%{CISCOFW402119}",
                        "cisco_message", "%{CISCOFW419001}",
                        "cisco_message", "%{CISCOFW419002}",
                        "cisco_message", "%{CISCOFW500004}",
                        "cisco_message", "%{CISCOFW602303_602304}",
                        "cisco_message", "%{CISCOFW710001_710002_710003_710005_710006}",
                        "cisco_message", "%{CISCOFW713172}",
                        "cisco_message", "%{CISCOFW733100}"
                ]
        }

#If CiscoTag is ASA-6-722051
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722051" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> IPv4 Address <%{IPV4:assigned_ip}%{GREEDYDATA:extra_field}"]
 }
}

  #If CiscoTag is ASA-6-722055
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-722055" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s<%{DATA:Policy}> User\s<%{DATA:[event_data][TargetUserName]}> IP\s<%{IPV4:src_ip}> Client Type:%{GREEDYDATA:VPN_Client}"]
 }
}

 #If CiscoTag is ASA-6-722055 - Authentication Rejected
 if [type] == "cisco-fw" and [ciscotag] == "ASA-6-113005" {
 grok {
 match => ["cisco_message", "%{DATA:Group}\s%{GREEDYDATA:Policy} : reason =\s%{DATA:Reason} : server =\s%{IPV4:ServerIP} : user =\s%{DATA:[event_data][TargetUserName]} : user IP =\s%{IPV4:src_ip}"]
 }
}

#if cisco tag is ASA-6-113004
if [type] == "cisco-fw" and [ciscotag] == "ASA-6-113004" {
grok {
match => ["cisco_message", "%{WORD:aaa_type} Successful : server =  %{IP:aaa_server_ip} : user = %{GREEDYDATA:user}"]
 }
}


#if CiscoTAG is ASA-6-734001
if [type] == "cisco-fw" and [ciscotag] == "%ASA-6-734001" {
grok {
match => ["cisco_message", "DAP: User %{DATA:user}, Addr %{IP:src_ip}, Connection %{DATA:protocol}: The following DAP records were selected for this connection: %{GREEDYDATA:policy_id1}"]
 }
}

# Parse the syslog severity and facility
        syslog_pri { }

if [type] == "cisco-fw" {
        geoip {
                add_tag => [ "GeoIP" ]
                database => "/opt/logstash/databases/GeoLite2-City.mmdb"
                source => "src_ip"
                add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
                add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
                }
       mutate {
                convert => [ "[geoip][coordinates]", "float"]
}
        if [geoip][city_name]      == "" { mutate { remove_field => "[geoip][city_name]" } }
        if [geoip][continent_code] == "" { mutate { remove_field => "[geoip][continent_code]" } }
        if [geoip][country_code2]  == "" { mutate { remove_field => "[geoip][country_code2]" } }
        if [geoip][country_code3]  == "" { mutate { remove_field => "[geoip][country_code3]" } }
        if [geoip][country_name]   == "" { mutate { remove_field => "[geoip][country_name]" } }
        if [geoip][latitude]       == "" { mutate { remove_field => "[geoip][latitude]" } }
        if [geoip][longitude]      == "" { mutate { remove_field => "[geoip][longitude]" } }
        if [geoip][postal_code]    == "" { mutate { remove_field => "[geoip][postal_code]" } }
        if [geoip][region_name]    == "" { mutate { remove_field => "[geoip][region_name]" } }
        if [geoip][time_zone]      == "" { mutate { remove_field => "[geoip][time_zone]" } }

        }

        # Gets the source IP whois information from the GeoIPASNum.dat flat file database
        geoip {
                add_tag => [ "Whois" ]
                database => "/opt/logstash/databases/GeoLite2-ASN.mmdb"
                source => "src_ip"
        }

        # Parse the date
        date {
                match => ["timestamp",
                        "MMM dd HH:mm:ss",
                        "MMM  d HH:mm:ss",
                        "MMM dd yyyy HH:mm:ss",
                        "MMM  d yyyy HH:mm:ss"
                ]
        }
}

output { stdout { codec => rubydebug } }

When I run a test this is what I get.

{
    "syslog_severity_code" => 6,
               "timestamp" => "May 08 2020 10:50:53",
                    "tags" => [
        [0] "_grokparsefailure",
        [1] "_geoip_lookup_failure"
    ],
    "syslog_facility_code" => 16,
                 "message" => "<134>May 08 2020 10:50:53: %ASA-6-734001: DAP: User xxxx.xxxxx, Addr xxx.xxx.xxx.xxx, Connection AnyConnect: The following DAP records were selected for this connection: XXXXXXXX",
              "@timestamp" => 2020-05-08T16:50:53.000Z,
                "ciscotag" => "ASA-6-734001",
              "syslog_pri" => "134",
         "syslog_facility" => "local0",
           "cisco_message" => "DAP: User xxxxxx.xxxx, Addr xxx.xx.x.x, Connection AnyConnect: The following DAP records were selected for this connection: XXXXXXXX",
                "@version" => "1",
                    "host" => "elk",
         "syslog_severity" => "informational"
}

I think the issue is with the time or "cisco_message" does in the grok filter.

I know there are other posts like this, but I cannot put it together. Any help would be wonderful. Thanks for reading my post.

Also here is the logstash start output incase it is needed.

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf --path.settings /etc/logstash/
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by com.headius.backport9.modules.Modules (file:/usr/share/logstash/logstash-core/lib/jars/jruby-complete-9.2.9.0.jar) to method sun.nio.ch.NativeThread.signal(long)
WARNING: Please consider reporting this to the maintainers of com.headius.backport9.modules.Modules
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
Sending Logstash logs to /usr/share/logstash/logs which is now configured via log4j2.properties
[2020-05-08T14:49:49,360][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2020-05-08T14:49:49,604][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"7.6.2"}
[2020-05-08T14:49:54,864][INFO ][org.reflections.Reflections] Reflections took 59 ms to scan 1 urls, producing 20 keys and 40 values
[2020-05-08T14:49:57,170][INFO ][logstash.filters.geoip   ][main] Using geoip database {:path=>"/opt/logstash/databases/GeoLite2-City.mmdb"}
[2020-05-08T14:49:57,305][INFO ][logstash.filters.geoip   ][main] Using geoip database {:path=>"/opt/logstash/databases/GeoLite2-ASN.mmdb"}
[2020-05-08T14:49:57,426][WARN ][org.logstash.instrument.metrics.gauge.LazyDelegatingGauge][main] A gauge metric of an unknown type (org.jruby.RubyArray) has been created for key: cluster_uuids. This may result in invalid serialization.  It is recommended to log an issue to the responsible developer/development team.
[2020-05-08T14:49:57,434][INFO ][logstash.javapipeline    ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>4, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>500, "pipeline.sources"=>["/etc/logstash/conf.d/test.conf"], :thread=>"#<Thread:0x662e86ff run>"}
[2020-05-08T14:50:00,678][INFO ][logstash.javapipeline    ][main] Pipeline started {"pipeline.id"=>"main"}
The stdin plugin is now waiting for input:
[2020-05-08T14:50:00,768][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2020-05-08T14:50:01,119][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600} it is needed.

Try this for your grok filter:

match => ["cisco_message", "DAP: User %{USERNAME:user}, Addr %{IP:src_ip}, Connection %{DATA:protocol}: The following DAP records were selected for this connection: %{GREEDYDATA:policy_id1}$"]

Ben

Thank you Ben!

This is what I have in the config now, what you posted.


#if CiscoTAG is ASA-6-734001
if [type] == "cisco-fw" and [ciscotag] == "ASA-6-734001" {
grok {
match => ["cisco_message", "DAP: User %{USERNAME:user}, Addr %{IP:src_ip}, Connection %{DATA:protocol}: The following DAP records were selected for this connection: %{GREEDYDATA:policy_id1}$"]
 }
}

This is the TEST output


{
         "syslog_severity" => "informational",
                "@version" => "1",
                    "host" => "elk",
                 "message" => "<134>May 08 2020 10:50:17: %ASA-6-734001: DAP: User ME.YOU, Addr xx.xx.xx.xx, Connection AnyConnect: The following DAP records were selected for this connection: XX_XXXX_XXX",
    "syslog_severity_code" => 6,
                    "tags" => [
        [0] "_grokparsefailure",
        [1] "_geoip_lookup_failure"
    ],
         "syslog_facility" => "local0",
              "syslog_pri" => "134",
              "@timestamp" => 2020-05-08T16:50:17.000Z,
                "ciscotag" => "ASA-6-734001",
               "timestamp" => "May 08 2020 10:50:17",
           "cisco_message" => "DAP: User ME.YOU, Addr xx.xxx.xx.xx, Connection AnyConnect: The following DAP records were selected for this connection: XX_XX_XXX",
    "syslog_facility_code" => 16
}

Also here are some Grok test outputs

Thanks to everyone who have looked at this!

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.