Dissectfailure when it shouldn't

[stillfreem]

Hi All,

I'm puzzled.

I have a logstash configuration working on two servers. First is fine, but on the second an error says there is a dissect failure

[2022-08-18T11:13:02,260][WARN ][org.logstash.dissect.Dissector] Dissector mapping, pattern not found {"field"=>"message", "pattern"=>"%{TZ} %{LogLevel}  %{LogCollector} %{Service}: [%{TrafficType}] %{Action}:   %{Module} %{Rule->}|[%{Signature}]|%{}|%{SignatureVersion}|%{Technique}", "event"=>{"tags"=>["_dissectfailure"], "host"=>"127.0.0.1", "@timestamp"=>2022-08-18T07:13:01.000Z, "@version"=>"1", "facility"=>1, "timestamp"=>"Aug 18 07:13:01", "logsource"=>"127.0.0.1", "severity_label"=>"Warning", "facility_label"=>"user-level", "priority"=>12, "program"=>"1/AME/vadcuda01/box_Firewall_threat", "severity"=>4, "message"=>"-04:00 Warning  vadcuda01 firewall: [Request] Allow:   PPP TCP 10.22.141.11 -> 10.130.120.187:3389 |[Unallowed Port Protocol Detected (SSL)]||0|\n"}}

As you can see my pattern matches the message, no idea why there is a dissect failure

Pattern

%{TZ} %{LogLevel}  %{LogCollector} %{Service}: [%{TrafficType}] %{Action}:   %{Module} %{Rule->}|[%{Signature}]|%{}|%{SignatureVersion}|%{Technique}

Message

-04:00 Warning  vadcuda01 firewall: [Request] Allow:   PPP TCP 10.22.141.12 -> 10.130.122.49:3389 |[Unallowed Port Protocol Detected (SSL)]||0|

Tested online here

[leandrojmp (Leandro Pereira)]

Please share your entire configuration, or at least until the dissect point, not just the pattern.

[stillfreem]

Thank you for the quick response @leandrojmp, bear in mind that the same configuration is working on server1 but pops the error on server2, thankyou in advance for your coop

input
{
#stdin{}
        syslog {
                host => "127.0.0.1"
                port => 5000
        }
}

filter
{
        if [program] =~ "box_Firewall_Activity" and [message] =~ "proto=" {
                grok {
                        keep_empty_captures => true
                        match => { "message" => "(?<Timezone>[\-\+][\d]{2}:[\d]{2})\s+%{WORD:LogLevel}\s+(?<OriginSource>[a-zA-Z0-9]+)\s+%{WORD:Action}:\s+%{GREEDYDATA:msg}" }
                }
                dissect { mapping => { "msg" => "type=%{type}|proto=%{proto}|srcIF=%{srcIF}|srcIP=%{srcIP}|srcPort=%{srcPort}|srcMAC=%{srcMAC}|dstIP=%{dstIP}|dstPort=%{dstPort}|dstService=%{dstService}|dstIF=%{dstIF}|rule=%{rule}|info=%{info}|srcNAT=%{srcNAT}|dstNAT=%{dstNAT}|duration=%{duration}|count=%{count}|receivedBytes=%{receivedBytes}|sentBytes=%{sentBytes}|receivedPackets=%{receivedPackets}|sentPackets=%{sentPackets}|user=%{user}|protocol=%{protocol}|application=%{application}|target=%{target}|content=%{content}|urlcat=%{urlcat}" } }
                mutate { add_tag => "commonFW" add_tag => "PH2"  add_tag => "kv" add_tag => "server" add_tag => "box_Firewall_Activity_kv" add_field => { "EventTime" => "%{@timestamp}" } }
                prune { blacklist_names => [ "message", "msg" ] }
                }
        else if [program] =~ "box_Firewall_Activity" and [message] !~ "proto=" {
                grok {
                        keep_empty_captures => true
                        match => { "message" => "(?<Timezone>\+[\d]{2}:[\d]{2})\s+%{WORD:LogLevel}\s+(?<OriginSource>[a-zA-Z0-9]+)\s+%{WORD:Action}:\s+%{GREEDYDATA:msg}" }
                }
               csv { source => "msg" separator => "|" columns => ["type", "proto", "srcIF", "srcIP", "srcPort", "srcMAC", "dstIP", "dstPort", "dstService", "dstIF", "rule", "info", "srcNAT", "dstNAT", "duration", "count", "receivedBytes", "sentBytes", "receivedPackets", "sentPackets", "user", "protocol", "application", "target", "content", "urlcat"] }
                mutate { add_tag => "commonFW" add_tag => "PH2"  add_tag => "csvPipe" add_tag => "server" add_tag => "box_Firewall_Activity_v" add_field => { "EventTime" => "%{@timestamp}" } }
                prune { blacklist_names => [ "message", "msg" ] }
        }
        else if [program] =~ "box_Firewall_threat" {
                dissect { mapping => { "message" => "%{TZ} %{LogLevel}  %{LogCollector} %{Service}: [%{TrafficType}] %{Action}:   %{Module} %{Rule->}|[%{Signature}]|%{}|%{SignatureVersion}|%{Technique}" } }
                mutate { add_tag => "ips" add_tag => "PH2"  add_tag => "server" add_tag => "box_Firewall_threat" add_field => { "EventTime" => "%{@timestamp}" } }
                prune { blacklist_names => [ "message", "msg" ] }
        } else { drop{} }

#       Blocks Inbound and blocked traffic from cudas
        if "commonFW" in [tags] and "cuda" in [OriginSource] {
                cidr {
                        add_tag => [ "iip" ]
                        address => [ "%{srcIP}" ]
                        network => [ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16" ]
                }
                if "iip" not in [tags] and ([Action] =~ "Block" or [Action] =~ "Denied" or [Action] =~ "Remove") { mutate { add_tag => "dropped inbound cuda" } drop{} }
                if "iip" not in [tags] and ([Action] !~ "Block" or [Action] !~ "Denied" or [Action] !~ "Remove") { mutate { add_tag => "allowed inbound cuda" } }
        } else if "commonFW" in [tags] and "ifw" in [OriginSource] { mutate {add_tag => "ifw" } }
#       Blocks IPv6 traffic
        if ":" in [srcIP] {drop{}}
}
output
{
#       if "iip" not in [tags] and "commonFW" in [tags] and "cuda" in [tags] and ([Action] =~ "Block" or [Action] =~ "Denied" or [Action] =~ "Remove") { exec { command => 'echo " %{OriginSource} blocked %{srcIP} %{Action} SHOULD NOT BE SEEN 1" ' } }
#       else if "iip" not in [tags] and "commonFW" in [tags] and "cuda" in [tags] and ([Action] !~ "Block" or [Action] !~ "Denied" or [Action] !~ "Remove") { exec { command => 'echo " %{OriginSource} %{srcIP} %{Action} 2" ' } }
#       else if "iip" in [tags] and "commonFW" in [tags] { exec { command => 'echo " %{OriginSource} %{srcIP} %{Action} 3" ' } }
#       else if "ifw" in [tags] and "commonFW" in [tags] { exec { command => 'echo %{OriginSource} %{srcIP} 4' } }
#       else if "ips" in [tags] { exec { command => 'echo "ips %{Signature} 5" ' } }
        if "commonFW" in [tags] {
                kusto {
                        path => "/tmp/kusto/%{+YYYY-MM-dd-HH-mm-ss}.txt"
                        ingest_url => "XXX"
                        app_id => "${W_APP_ID}"
                        app_key => "${W_APP_KEY}"
                        app_tenant => "${W_APP_TENANT}"
                        database => "DB"
                        table => "table"
                        json_mapping => "mapping"
                }
        } else if "ips" in [tags] {
                kusto {
                        path => "/tmp/kusto/%{+YYYY-MM-dd-HH-mm-ss}.txt"
                        ingest_url => "XXX"
                        app_id => "${W_APP_ID}"
                        app_key => "${W_APP_KEY}"
                        app_tenant => "${W_APP_TENANT}"
                        database => "DB"
                        table => "table"
                        json_mapping => "mapping"
                }
        }
}

[stillfreem]

Hey @leandrojmp any luch?