Error in copying fields

I have created the logstash conf file properly and there is no error. My problem is I am unable to copy , split the fields properly. In the below log

<185>Jul 30 2019 10:15:13 BR-RTS-BHB-T3-VM-NCN-BST-US57-01-BAB IFNET/1/IF_LINKDOWN:OID 1.3.6.1.6.3.1.1.5.3 Interface 11 turned into DOWN state.(AdminStatus=1,OperStatus=2,InterfaceName=GigabitEthernet0/0/7)

I am able to filter the

IFNET/1/IF_LINKDOWN

to a seperate field SType
I need the 1 to be separated to a field Alertlevel and later translate to Alertlevel_desc.
I was able to do all the above steps. But my requirement is to copy SType into a separate field SyslogID(I need the original(before splitting) value of SType also). I was unable to generate SyslogID in the output of logstash. Kindly request you to tell me the solution for this? Please look into the logstash conf and output below.

input {
   file {
	path => ["/test_huawei.log"]
	start_position => "beginning"
    }	
}


filter
{
      grok
	{

	patterns_dir => ["./patterns"]
	break_on_match => true



	match => [ "message", "%{SYSLOGTIMESTAMP:date}\s%{NOTSPACE:ISE-Hostname}\s%{NOTSPACE:Name}\s%{NUMBER:val1}\s%{NUMBER:val2}\s%{NUMBER:val3}\s%{TIMESTAMP_ISO8601:timestamp}\s%{NOTSPACE:timezone}\s%{NUMBER:value4}\s%{NUMBER:value5}\s%{WORD:LogType}\s%{NOTSPACE:AccName}:\s%{DATA:Alias},\s(ConfigVersionId=%{NUMBER:ConfigVersionID}),\s(Device IP Address=%{IPORHOST:DeviceIP}),\s(RequestLatency=%{INT:RequestLatency}),\s(NetworkDeviceName=%{IPORHOST:NetworkDeviceName}),\s(User-Name=%{NUMBER:User-Name}),\s(NAS-IP-Address=%{IPORHOST:NAS-IP-Address}),\s(NAS-Port=%{NUMBER:NAS-Port}),\s(Framed-Protocol=%{WORD:Framed-Protocol}),\s(Framed-IP-Address=%{IPORHOST:Framed-IP-Address}),\s(Class=%{NOTSPACE:Class}),\s(Calling-Station-ID=%{NOTSPACE:Calling-StationID}),\s(NAS-Identifier=%{NOTSPACE:NAS-Identifier}),\s(Acct-Status-Type=%{WORD:Acct-Status-Type}),\s(Acct-Session-Id=%{NOTSPACE:Acct-Session-Id}),\s(Acct-Authentic=%{WORD:Acct-Authentic}),\s(Event-Timestamp=%{NUMBER:Event-Timestamp}),\s(NAS-Port-Type=%{WORD:NAS-Port-Type}),\s(NAS-Port-Id=%{NOTSPACE:NAS-Port-Id}),\s(AcsSessionID=%{NOTSPACE:AcsSessionID}),\s(SelectedAccessService=%{NOTSPACE:SelectedAccessService})\s%{GREEDYDATA:infolog}" ]

	match =>  [ "message", "%{SYSLOGTIMESTAMP:datetime}:%{SPACE}%{NOTSPACE:SType}:%{SPACE}%{WORD:direction}%{SPACE}%{IPORHOST:src_ip}%{SPACE}%{NOTSPACE:Status}%{SPACE}%{NOTSPACE:values}%{SPACE}%{GREEDYDATA:info}" ]
	
	match => [ "message", "%{SYSLOGTIMESTAMP:datetime}:%{SPACE}%{NOTSPACE:SType}:%{SPACE}%{GREEDYDATA:info}" ]
	
	match => [ "message", "%{SYSLOGTIMESTAMP:datetime}%{SPACE}%{WORD:ZONE}:%{SPACE}%{NOTSPACE:SType}:%{SPACE}%{NOTSPACE:subtypes}%{GREEDYDATA:info}" ]

	match => [ "message", "%{SYSLOGTIMESTAMP:datetime}\s%{NOTSPACE:Timezone}:\s%{NOTSPACE:Host}:\s%{GREEDYDATA:info}" ]
	match => [ "message", "\S%{POSINT:NO}\S%{POSINT:SyslogTag}[:]\s{1,}%{GREEDYDATA:info}" ]
	add_field => [ "received_at", "%{@timestamp}" ]
        add_field => [ "received_from", "%{host}" ]

	  overwrite => [ "message" ]

    }
       if ![SType] {
       		grok {
			match => { "message" => "\S%{POSINT}\S(?<datetime>%{MONTH} +%{MONTHDAY} %{YEAR} %{TIME})(.*?[-])*%{NOTSPACE}[%]?[%]?%{DATA:SType}[:\(]"}
#			add_field => [ "received_at", "%{@timestamp}" ]
#			add_field => [ "received_from", "%{host}" ]
		}

	}
#    translate
#        {
#        field => "[SType]"
#        destination => "[SyslogId]"
#	}

    mutate
	{
	copy => { "SType" => "SyslogId" }
	gsub => ["SyslogId","/","-"]
#	copy => { "Stype" => "SType" } 
#	split => ["Stype","-"]
	split => ["SyslogId","-"]
	add_field => { "Alertlevel" => "%{[SyslogId][1]}" }
	}
    translate
	{
	field => "[Alertlevel]"
	destination => "[Alertlevel_desc]"
	dictionary => {
		"0" => "Emerg"
		"1" => "Alert"
		"2" => "Critical"
		"3" => "Error"
		"4" => "Warnings"
		"5" => "Notifications"
		"6" => "Informational"
		"7" => "Debugging"
	}
   }
}

output {
	 elasticsearch {
                hosts => ["127.0.0.1:9200"]
                index => "testsys1"
        }

	stdout { codec => rubydebug }
}

Output below:

"@timestamp" => 2019-07-31T12:47:38.288Z,
               "tags" => [
        [0] "_grokparsefailure"
    ],
           "@version" => "1",
              "SType" => [
        [0] " LLDP",
        [1] "4",
        [2] "NBRCHGTRAP"
    ],
         "Alertlevel" => "4",
               "host" => "localhost",
    "Alertlevel_desc" => "Warnings",
            "message" => "<188>Jul 30 2019 10:19:19 UL-DDN-DDN-T3-TT-NCN-BST-US57-01-DDN LLDP/4/NBRCHGTRAP:OID: 1.0.8802.1.1.2.0.0.1 Neighbor information is changed. (LldpStatsRemTablesInserts=0, LldpStatsRemTablesDeletes=0, LldpStatsRemTablesDrops=21, LldpStatsRemTablesAgeouts=0)",
               "path" => "/test_huawei.log",
           "datetime" => "Jul 30 2019 10:19:19"

Here I want to have SyslogId with value LLDP/4/NBRCHGTRAP . How can I do this????

I find it hard to believe you get that output for that message with that configuration. I get

  "datetime" => "Jul 30 2019 10:19:19",
  "SyslogId" => " LLDP/4/NBRCHGTRAP",
     "SType" => " LLDP/4/NBRCHGTRAP",
   "message" => "<188>Jul 30 2019 10:19:19 UL-DDN-DDN-T3-TT-NCN-BST-US57-01-DDN LLDP/4/NBRCHGTRAP:OID: 1.0.8802.1.1.2.0.0.1 Neighbor information is changed. (LldpStatsRemTablesInserts=0, LldpStatsRemTablesDeletes=0, LldpStatsRemTablesDrops=21, LldpStatsRemTablesAgeouts=0)",
"Alertlevel" => "%{[SyslogId][1]}",
      "tags" => [
    [0] "_jsonparsefailure",
    [1] "_grokparsefailure"
]

The mutate filter does operations in a fixed order, and copy comes after gsub, so you need to break the mutate in two...

mutate {
    copy => { "SType" => "SyslogId" }
}
mutate {
    gsub => ["SyslogId","/","-"]
    ...

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.