Same logstash config file give different result for the same input on different machines

Hi I have created one logstash conf file but I am running the same config file on two machine, one is mac and other is ubbuntu.

For the same input data on my mac I am getting the correct output where as in ubuntu I am getting _grokparsefailure.

I am not sure why this is happening

MAC

2020-08-20 11:25:33.301770528 re0:app:1 APP Function = "function_name", IP = "1.1.1.2/32", Msg = "OnAdd"
{
"match_2" => "OnAdd",
"traceType" => "APP",
"ipv4Address" => "1.1.1.2",
"functionName" => "function_name",
"timestamp" => "2020-08-20 11:25:33.301770528",
"match_1" => "IP = "1.1.1.2/32", Msg = "OnAdd"",
"@timestamp" => 2021-02-09T16:44:32.976Z,
"message" => "2020-08-20 11:25:33.301770528 re0:app:1 APP Function = "function_name", IP = "1.1.1.2/32", Msg = "OnAdd"",
"host" => "mac",
"tags" => [
[0] "_grokparsefailure"
],
"pid" => "1",
"program" => "app",
"node" => "re0",
"@version" => "1"
}

UBUNTU

2020-08-20 11:25:33.301770528 re0:app:1 APP Function = "function_name", IP = "1.1.1.2/32", Msg = "OnAdd"
{
"program" => "app",
"host" => "ubuntu",
"timestamp" => "2020-08-20 11:25:33.301770528",
"tags" => [
[0] "_grokparsefailure"
],
"node" => "re0",
"pid" => "1",
"match_1" => "IP = "1.1.1.2/32", Msg = "OnAdd"",
"functionName" => "function_name",
"@version" => "1",
"traceType" => "APP",
"@timestamp" => 2021-02-09T16:39:03.941Z,
"message" => "2020-08-20 11:25:33.301770528 re0:app:1 APP Function = "function_name", IP = "1.1.1.2/32", Msg = "OnAdd""

What is your logstash config? You need to share your pipeline config.

The tag _grokparsefailure appears in both messages you pasted in your question.

input {
stdin { }
}

filter {
    if "Function =" in [message] {
            grok {
                    break_on_match => false
                    pattern_definitions => {
                            "CUST_MSG" => "((Msg|message|Message|message1|message2|Message1|Message2) [=])"
                            "CUST_IFNAME" => "((Interface|Ifl_name|L2Interface|IFD_Name) [=])"
                            "CUST_STATE" => "(State [=])"
                            "CUST_NEXTHOP" => "((NexthopId|Index) [=])"
                            "CUST_RETRY" => "(Retry [=])"
                            "CUST_CLONECNT" => "(CloneRefCnt [=])"
                            "CUST_TID" => "(TableId [=])"
                            "CUST_BDI" => "(BDIdx [=])"
                    }
                    match => {
                            "message" => ["%{TIMESTAMP_ISO8601:timestamp} %{WORD:node}:%{WORD:program}:%{INT:pid} %{WORD:traceType}  Function = \"%{WORD:functionName}\", %{GREEDYDATA:match_1}"]
                            "match_1" => ["%{CUST_MSG} \"%{DATA:match_2}\"" ,"%{IPV4:ipv4Address}", "%{IPV6:ipv6Address}", "%{MAC:macAddress}",
                                            "%{CUST_IFNAME} \"%{DATA:ifname}\"" ,
                                            "%{CUST_STATE} \"%{DATA:state}\"" ,
                                            "%{CUST_NEXTHOP} %{INT:nhid}",
                                            "%{CUST_RETRY} %{INT:retryCount}",
                                            "%{CUST_CLONECNT} %{INT:cloneRefCount}",
                                            "%{CUST_TID} %{INT:tableId}",
                                            "%{CUST_BDI} %{INT:bdIndex}"]
                            "match_2" => ["%{WORD:arpType}, %{IP:targetIp}, %{MAC:targetMac}, %{IP:sourceIp}, %{MAC:sourceMac}"]
                    }
            }
            grok {

            }
    } else {
            drop { }
    }

    if "_grokparsefailure" in [tags] {
            #drop { }
    }
}

output {
stdout { } 

}

Output with same config file.

Non working case ubuntu

Using bundled JDK: /usr/share/logstash/jdk
logstash 7.10.2

2020-08-20 11:25:33.299621422 re0:arpd:13042 ARP_TP_ARP_ENTRY_MSG Function = "arprequest", Msg = "Sending Req", Interface = "et-1/0/0:0.0", L2Interface = "0", BDIdx = 0, TableId = 0, TargetIp = "1.1.1.2", TargetMac = "00:00:00:00:00:00", State = "Refresh", Retry = 4, NexthopId = 41002, CloneRefCnt = 0

{
"program" => "arpd",
"host" => "host2",
"timestamp" => "2020-08-20 11:25:33.299621422",
"tags" => [
[0] "_grokparsefailure"
],
"node" => "re0",
"pid" => "13042",
"match_1" => "Msg = "Sending Req", Interface = "et-1/0/0:0.0", L2Interface = "0", BDIdx = 0, TableId = 0, TargetIp = "1.1.1.2", TargetMac = "00:00:00:00:00:00", State = "Refresh", Retry = 4, NexthopId = 41002, CloneRefCnt = 0",
"functionName" => "arprequest",
"@version" => "1",
"traceType" => "ARP_TP_ARP_ENTRY_MSG",
"@timestamp" => 2021-02-09T17:28:31.593Z,
"message" => "2020-08-20 11:25:33.299621422 re0:arpd:13042 ARP_TP_ARP_ENTRY_MSG Function = "arprequest", Msg = "Sending Req", Interface = "et-1/0/0:0.0", L2Interface = "0", BDIdx = 0, TableId = 0, TargetIp = "1.1.1.2", TargetMac = "00:00:00:00:00:00", State = "Refresh", Retry = 4, NexthopId = 41002, CloneRefCnt = 0"
}

In working case

logstash --version
logstash 7.9.0

2020-08-20 11:25:33.299621422 re0:arpd:13042 ARP_TP_ARP_ENTRY_MSG Function = "arprequest", Msg = "Sending Req", Interface = "et-1/0/0:0.0", L2Interface = "0", BDIdx = 0, TableId = 0, TargetIp = "1.1.1.2", TargetMac = "00:00:00:00:00:00", State = "Refresh", Retry = 4, NexthopId = 41002, CloneRefCnt = 0
{
"@timestamp" => 2021-02-09T17:29:32.867Z,
"host" => "host,
"match_1" => "Msg = "Sending Req", Interface = "et-1/0/0:0.0", L2Interface = "0", BDIdx = 0, TableId = 0, TargetIp = "1.1.1.2", TargetMac = "00:00:00:00:00:00", State = "Refresh", Retry = 4, NexthopId = 41002, CloneRefCnt = 0",
"message" => "}2020-08-20 11:25:33.299621422 re0:arpd:13042 ARP_TP_ARP_ENTRY_MSG Function = "arprequest", Msg = "Sending Req", Interface = "et-1/0/0:0.0", L2Interface = "0", BDIdx = 0, TableId = 0, TargetIp = "1.1.1.2", TargetMac = "00:00:00:00:00:00", State = "Refresh", Retry = 4, NexthopId = 41002, CloneRefCnt = 0",
"ipv4Address" => "1.1.1.2",
"retryCount" => "4",
"functionName" => "arprequest",
"state" => "Refresh",
"pid" => "13042",
"match_2" => "Sending Req",
"@version" => "1",
"cloneRefCount" => "0",
"node" => "re0",
"macAddress" => "00:00:00:00:00:00",
"ifname" => "et-1/0/0:0.0",
"nhid" => "41002",
"program" => "arpd",
"bdIndex" => "0",
"timestamp" => "2020-08-20 11:25:33.299621422",
"traceType" => "ARP_TP_ARP_ENTRY_MSG",
"tags" => [
[0] "_grokparsefailure"
],
"tableId" => "0"
}

I cannot speak to why it would work on one platform but not another, but I can make two points:

The empty grok will always add a _grokparsefailure tag

grok {}

If you separate the match_1 and match_2 patterns into a separate filter then it works a little better. You may need 3 filters.

       grok {
           match => {
                        "message" => ["%{TIMESTAMP_ISO8601:timestamp} %{WORD:node}:%{WORD:program}:%{INT:pid} %{WORD:traceType} Function = \"%{WORD:functionName}\", %{GREEDYDATA:match_1}"]
           }
       }
       grok {
                break_on_match => false
                pattern_definitions => {
                        "CUST_MSG" => "((Msg|message|Message|message1|message2|Message1|Message2) [=])"
                        "CUST_IFNAME" => "((Interface|Ifl_name|L2Interface|IFD_Name) [=])"
                        "CUST_STATE" => "(State [=])"
                        "CUST_NEXTHOP" => "((NexthopId|Index) [=])"
                        "CUST_RETRY" => "(Retry [=])"
                        "CUST_CLONECNT" => "(CloneRefCnt [=])"
                        "CUST_TID" => "(TableId [=])"
                        "CUST_BDI" => "(BDIdx [=])"
                }
                match => {
                        "match_1" => ["%{CUST_MSG} \"%{DATA:match_2}\"" ,"%{IPV4:ipv4Address}", "%{IPV6:ipv6Address}", "%{MAC:macAddress}",
                                        "%{CUST_IFNAME} \"%{DATA:ifname}\"" ,
                                        "%{CUST_STATE} \"%{DATA:state}\"" ,
                                        "%{CUST_NEXTHOP} %{INT:nhid}",
                                        "%{CUST_RETRY} %{INT:retryCount}",
                                        "%{CUST_CLONECNT} %{INT:cloneRefCount}",
                                        "%{CUST_TID} %{INT:tableId}",
                                        "%{CUST_BDI} %{INT:bdIndex}"]
                        "match_2" => ["%{WORD:arpType}, %{IP:targetIp}, %{MAC:targetMac}, %{IP:sourceIp}, %{MAC:sourceMac}"]
                }
        }