I need to help on the parsing of my logs.
This is my parser:
input{
tcp {
port => "5140"
tags => "syslog"
}
}
filter {
grok {
match => { "message" => "%{SYSLOG5424PRI:syslog_index}%{SYSLOGHOST:syslog_host} %{GREEDYDATA:message}"}
}
json {source => "message"}
}
output {
stdout { codec => rubydebug }
elasticsearch {
hosts => ["https://xxxx:9200", "https://yyyy:9200"]
user => "elastic"
password => "kkkk"
cacert => "/etc/logstash/certs/ca.crt"
index => "fortinet-%{+YYYY.MM.dd}"
action => "index"
}
}
This is the logs that i receive in logstash:
at [Source: (byte)"<01>- hostname {"name":"DefaultProfile","version":"1.0","isoTimeFormat":"yyyy-MM-dd'T'HH:mm:ss.SSSZ","type":"Event","category":"accept","protocolID":"6","sev":"0","src":"10.38.10.25","dst":"10.0.0.254","srcPort":"42867","dstPort":"1105","relevance":"5","credibility":"5","startTimeEpoch":"1605700499872","startTimeISO":"2020-11-18T12:54:59.872+01:00","storageTimeEpoch":"1605700499872","storageTimeISO":"2020-11-18T12:54:59.872+01:00","deploymentID":"5c15c102-a647-11ea-8226-00505601062b","devTimeEpo"[truncated 1599 bytes]; line: 1, column: 2]>}
[2020-11-18T12:55:00,164][WARN ][logstash.filters.json ][main][4e9dd3386de5dd84ea83e51639a38dfe7c2e2993c1acce700ace95b9a1d3fa8a] Error parsing json {:source=>"message", :raw=>["<01>- hostname {"name":"DefaultProfile","version":"1.0","isoTimeFormat":"yyyy-MM-dd'T'HH:mm:ss.SSSZ","type":"Event","category":"close","protocolID":"6","sev":"0","src":"10.204.91.242","dst":"172.31.15.236","srcPort":"41940","dstPort":"12489","relevance":"5","credibility":"5","startTimeEpoch":"1605700500882","startTimeISO":"2020-11-18T12:55:00.882+01:00","storageTimeEpoch":"1605700500882","storageTimeISO":"2020-11-18T12:55:00.882+01:00","deploymentID":"5c15c102-a647-11ea-8226-00505601062b","devTimeEpoch":"1605712588000","devTimeISO":"2020-11-18T16:16:28.000+01:00","srcPreNATPort":"0","dstPreNATPort":"0","srcPostNATPort":"0","dstPostNATPort":"0","hasIdentity":"false","payload":"<173>Nov 18 17:14:37 1.1.1.3 date=2020-11-18 time=16:16:28 devname=CLD2008FW devid=FG3K0B3I12700629 logid=0000000013 type=traffic subtype=forward level=notice vd=cfs-adm srcip=10.204.91.242 srcport=41940 srcintf=SFR_SI_SUP_Back dstip=172.31.15.236 dstport=12489 dstintf=pfdev-int-admin poluuid=50afc926-ecb8-51e6-dc63-a45bd626e649 sessionid=2772748167 proto=6 action=close policyid=63 dstcountry=Reserved srccountry=Reserved trandisp=noop service=tcp/12489 duration=1 sentbyte=164 rcvdbyte=112 sentpkt=3 rcvdpkt=2 appcat=unscanned\n","eventCnt":"1","srcIPLoc":"Europe.France","dstIPLoc":"Europe.France","hasOffense":"false","domainID":"1","domainName":"Test-Tenant-1","eventName":"Forward Traffic","lowLevelCategory":"Firewall Permit","highLevelCategory":"Access","eventDescription":"Forward Traffic","protocolName":"tcp","logSource":"FortiGate @ 1.1.1.3","srcNetName":"Net-10-172-192.Net_10_0_0_0","dstNetName":"Net-10-172-192.Net_172_16_0_0","logSourceType":"Fortinet FortiGate Security Gateway","logSourceGroup":"Production,IPS","logSourceIdentifier":"1.1.1.3","BytesReceived":"112","BytesSent":"164"}", " date=2020-11-18 time=16:16:28 devname=CLD2008FW devid=FG3K0B3I12700629 logid=0000000013 type=traffic subtype=forward level=notice vd=cfs-adm srcip=10.204.91.242 srcport=41940 srcintf=SFR_SI_SUP_Back dstip=172.31.15.236 dstport=12489 dstintf=pfdev-int-admin poluuid=50afc926-ecb8-51e6-dc63-a45bd626e649 sessionid=2772748167 proto=6 action=close policyid=63 dstcountry=Reserved srccountry=Reserved trandisp=noop service=tcp/12489 duration=1 sentbyte=164 rcvdbyte=112 sentpkt=3 rcvdpkt=2 appcat=unscanned\n","eventCnt":"1","srcIPLoc":"Europe.France","dstIPLoc":"Europe.France","hasOffense":"false","domainID":"1","domainName":"Test-Tenant-1","eventName":"Forward Traffic","lowLevelCategory":"Firewall Permit","highLevelCategory":"Access","eventDescription":"Forward Traffic","protocolName":"tcp","logSource":"FortiGate @ 1.1.1.3","srcNetName":"Net-10-172-192.Net_10_0_0_0","dstNetName":"Net-10-172-192.Net_172_16_0_0","logSourceType":"Fortinet FortiGate Security Gateway","logSourceGroup":"Production,IPS","logSourceIdentifier":"1.1.1.3","BytesReceived":"112","BytesSent":"164"}"], :exception=>java.lang.ClassCastException: class org.jruby.RubyArray cannot be cast to class org.jruby.RubyIO (org.jruby.RubyArray and org.jruby.RubyIO are in unnamed module of loader 'app')}
[2020-11-18T12:55:00,965][WARN ][logstash.filters.json ][main][4e9dd3386de5dd84ea83e51639a38dfe7c2e2993c1acce700ace95b9a1d3fa8a] Error parsing json {:source=>"message", :raw=>"<01>- hostname {"name":"DefaultProfile","version":"1.0","isoTimeFormat":"yyyy-MM-dd'T'HH:mm:ss.SSSZ","type":"Event","category":"accept","protocolID":"6","sev":"0","src":"10.38.10.25","dst":"10.0.0.254","srcPort":"42867","dstPort":"1106","relevance":"5","credibility":"5","startTimeEpoch":"1605700501893","startTimeISO":"2020-11-18T12:55:01.893+01:00","storageTimeEpoch":"1605700501893","storageTimeISO":"2020-11-18T12:55:01.893+01:00","deploymentID":"5c15c102-a647-11ea-8226-00505601062b","devTimeEpoch":"1605713400000","devTimeISO":"2020-11-18T16:30:00.000+01:00","srcPreNAT":"10.38.10.25","srcPostNAT":"192.168.0.254","srcPreNATPort":"-22669","dstPreNATPort":"0","srcPostNATPort":"-22669","dstPostNATPort":"0","hasIdentity":"false","payload":"Nov 18 17:14:37 1.1.1.3 logver=60 timestamp=1602513000 tz=UTC+2:00 devname=Test-Port-Scan devid=FG1K5D3I17802467 vd=SCAN date=2020-11-18 time=16:30:00 logid=0000000020 type=traffic subtype=forward level=notice eventtime=1602511106 srcip=10.38.10.25 srcport=42867 srcintf=SCAN-SRC srcintfrole=lan dstip=10.0.0.254 dstport=1106 dstintf=SCAN-DEST dstintfrole=lan poluuid=5d76d0ec-8784-51e8-0477-b1236fe2c2e1 sessionid=1001196239 proto=6 action=accept policyid=1 policytype=policy service=tcp/1106 dstcountry=United States srccountry=Reserved trandisp=snat transip=192.168.0.254 transport=42867 appid=43714 app=Test.scan appcat=General.Interest apprisk=low applist=default duration=10929 sentbyte=1258 rcvdbyte=3585 sentpkt=19 rcvdpkt=26 sentdelta=132 rcvddelta=483\n","eventCnt":"1","srcIPLoc":"Europe.France","dstIPLoc":"Europe.France","hasOffense":"false","domainID":"1","domainName":"Test-Tenant-1","eventName":"Firewall Permit","lowLevelCategory":"Firewall Permit","highLevelCategory":"Access","eventDescription":"Firewall Permit","protocolName":"tcp","logSource":"FortiGate @ 1.1.1.3","srcNetName":"Net-10-172-192.scan_src","dstNetName":"Net-10-172-192.scan_dest","logSourceType":"Fortinet FortiGate Security Gateway","logSourceGroup":"Production,IPS","logSourceIdentifier":"1.1.1.3","BytesReceived":"3585","BytesSent":"1258","Application":"Test.scan"}", :exception=>#<LogStash::Json::ParserError: Unexpected character ('<' (code 60)): expected a valid value (number, String, array, object, 'true', 'false' or 'null')
at [Source: (byte)"<01>- hostname {"name":"DefaultProfile","version":"1.0","isoTimeFormat":"yyyy-MM-dd'T'HH:mm:ss.SSSZ","type":"Event","category":"accept","protocolID":"6","sev":"0","src":"10.38.10.25","dst":"10.0.0.254","srcPort":"42867","dstPort":"1106","relevance":"5","credibility":"5","startTimeEpoch":"1605700501893","startTimeISO":"2020-11-18T12:55:01.893+01:00","storageTimeEpoch":"1605700501893","storageTimeISO":"2020-11-18T12:55:01.893+01:00","deploymentID":"5c15c102-a647-11ea-8226-00505601062b","devTimeEpo"[truncated 1599 bytes]; line: 1, column: 2]>}
Regards,