Hi Guys,
I am trying to parse Json formatted messages certain messages are getting parsed properly; while certain are showing as jsonparsefailure. Then I checked the same message online json viewer and it gets parsed perfectly but logstash refused to parse. Any clue why?
Here is the original message
{"transaction":{"client_ip":"192.168.5.76","time_stamp":"Tue Mar 31 18:46:11 2020","server_id":"023a162ad8c7afb2e1d2db424a6741ad78f46987","client_port":61492,"host_ip":"192.168.5.187","host_port":80,"unique_id":"158566057127.626744","request":{"method":"GET","http_version":1.1,"uri":"/aphpfilethatdonotexist.php?=../../etc/passwd","headers":{"Host":"192.168.5.187","Connection":"keep-alive","DNT":"1","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36","Upgrade-Insecure-Requests":"1","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9","Accept-Encoding":"gzip, deflate","Accept-Language":"en-GB,en-US;q=0.9,en;q=0.8,hi;q=0.7"}},"response":{"body":"<html>\r\n<head><title>403 Forbidden</title></head>\r\n<body>\r\n<center><h1>403 Forbidden</h1></center>\r\n<hr><center>nginx/1.17.9</center>\r\n</body>\r\n</html>\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n<!-- a padding to disable MSIE and Chrome friendly error page -->\r\n","http_code":403,"headers":{"Server":"nginx/1.17.9","Date":"Tue, 31 Mar 2020 13:16:11 GMT","Content-Length":"555","Content-Type":"text/html","Connection":"keep-alive"}},"producer":{"modsecurity":"ModSecurity v3.0.4 (Linux)","connector":"ModSecurity-nginx v1.0.1","secrules_engine":"Enabled","components":["OWASP_CRS/3.0.2\""]},"messages":[{"message":"Host header is a numeric IP address","details":{"match":"Matched \"Operator `Rx' with parameter `^[\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `192.168.5.187' )","reference":"o0,13v65,13","ruleId":"920350","file":"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf","lineNumber":"777","data":"192.168.5.187","severity":"4","ver":"OWASP_CRS/3.0.0","rev":"2","tags":["application-multi","language-multi","platform-multi","attack-protocol","OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST","WASCTC/WASC-21","OWASP_TOP_10/A7","PCI/6.5.10"],"maturity":"9","accuracy":"9"}},{"message":"Path Traversal Attack (/../)","details":{"match":"Matched \"Operator `Rx' with parameter `(?i)(?:\\x5c|(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5 (400 characters omitted)' against variable `REQUEST_URI_RAW' (Value: `/aphpfilethatdonotexist.php?=../../etc/passwd' )","reference":"o31,4v4,45","ruleId":"930100","file":"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf","lineNumber":"29","data":"Matched Data: /../ found within REQUEST_URI_RAW: /aphpfilethatdonotexist.php?=../../etc/passwd","severity":"2","ver":"OWASP_CRS/3.0.0","rev":"3","tags":["application-multi","language-multi","platform-multi","attack-lfi","OWASP_CRS/WEB_ATTACK/DIR_TRAVERSAL"],"maturity":"9","accuracy":"7"}},{"message":"Path Traversal Attack (/../)","details":{"match":"Matched \"Operator `Pm' with parameter `..\\ ../' against variable `REQUEST_URI' (Value: `/aphpfilethatdonotexist.php?=../../etc/passwd' )","reference":"o29,3v4,45","ruleId":"930110","file":"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf","lineNumber":"55","data":"Matched Data: ../ found within REQUEST_URI: /aphpfilethatdonotexist.php?=../../etc/passwd","severity":"2","ver":"OWASP_CRS/3.0.0","rev":"1","tags":[],"maturity":"9","accuracy":"7"}},{"message":"OS File Access Attempt","details":{"match":"Matched \"Operator `PmFromFile' with parameter `lfi-os-files.data' against variable `ARGS:' (Value: `../../etc/passwd' )","reference":"o6,10v33,16t:utf8toUnicode,t:urlDecodeUni,t:normalizePathWin,t:lowercase","ruleId":"930120","file":"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf","lineNumber":"84","data":"Matched Data: etc/passwd found within ARGS:: ../../etc/passwd","severity":"2","ver":"OWASP_CRS/3.0.0","rev":"4","tags":["application-multi","language-multi","platform-multi","attack-lfi","OWASP_CRS/WEB_ATTACK/FILE_INJECTION","WASCTC/WASC-33","OWASP_TOP_10/A4","PCI/6.5.4"],"maturity":"9","accuracy":"9"}},{"message":"Remote Command Execution: Unix Shell Code Found","details":{"match":"Matched \"Operator `PmFromFile' with parameter `unix-shell.data' against variable `ARGS:' (Value: `../../etc/passwd' )","reference":"o6,10v33,16t:urlDecodeUni,t:cmdLine,t:normalizePath,t:lowercase","ruleId":"932160","file":"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-932-APPLICATION-ATTACK-RCE.conf","lineNumber":"423","data":"Matched Data: etc/passwd found within ARGS:: ../../etc/passwd","severity":"2","ver":"OWASP_CRS/3.0.0","rev":"1","tags":["application-multi","language-shell","platform-unix","attack-rce","OWASP_CRS/WEB_ATTACK/COMMAND_INJECTION","WASCTC/WASC-31","OWASP_TOP_10/A1","PCI/6.5.2"],"maturity":"1","accuracy":"8"}},{"message":"Inbound Anomaly Score Exceeded (Total Score: 23)","details":{"match":"Matched \"Operator `Ge' with parameter `5' against variable `TX:ANOMALY_SCORE' (Value: `23' )","reference":"","ruleId":"949110","file":"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-949-BLOCKING-EVALUATION.conf","lineNumber":"44","data":"","severity":"2","ver":"","rev":"","tags":["application-multi","language-multi","platform-multi","attack-generic"],"maturity":"0","accuracy":"0"}}]}}
And logstash parser is
input {
stdin {
type => "json"
codec => "json"
}
}
filter{
json{
source => "message"
}
}
output {
stdout {}
}
And here it parses incorrectly though message is in json format -
"type" => "json",
"tags" => [
[0] "_jsonparsefailure"
],
"host" => "traplox",
"message" => "{\"transaction\":{\"client_ip\":\"192.168.5.76\",\"time_stamp\":\"Tue Mar 31 18:46:11 2020\",\"server_id\":\"023a162ad8c7afb2e1d2db424a6741ad78f46987\",\"client_port\":61492,\"host_ip\":\"192.168.5.187\",\"host_port\":80,\"unique_id\":\"158566057127.626744\",\"request\":{\"method\":\"GET\",\"http_version\":1.1,\"uri\":\"/aphpfilethatdonotexist.php?=../../etc/passwd\",\"headers\":{\"Host\":\"192.168.5.187\",\"Connection\":\"keep-alive\",\"DNT\":\"1\",\"User-Agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36\",\"Upgrade-Insecure-Requests\":\"1\",\"Accept\":\"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\",\"Accept-Encoding\":\"gzip, deflate\",\"Accept-Language\":\"en-GB,en-US;q=0.9,en;q=0.8,hi;q=0.7\"}},\"response\":{\"body\":\"<html>\\r\\n<head><title>403 Forbidden</title></head>\\r\\n<body>\\r\\n<center><h1>403 Forbidden</h1></center>\\r\\n<hr><center>nginx/1.17.9</center>\\r\\n</body>\\r\\n</html>\\r\\n<!-- a padding to disable MSIE and Chrome friendly error page -->\\r\\n<!-- a padding to disable MSIE and Chrome friendly error page -->\\r\\n<!-- a padding to disable MSIE and Chrome friendly error page -->\\r\\n<!-- a padding to disable MSIE and Chrome friendly error page -->\\r\\n<!-- a padding to disable MSIE and Chrome friendly error page -->\\r\\n<!-- a padding to disable MSIE and Chrome friendly error page -->\\r\\n\",\"http_code\":403,\"headers\":{\"Server\":\"nginx/1.17.9\",\"Date\":\"Tue, 31 Mar 2020 13:16:11 GMT\",\"Content-Length\":\"555\",\"Content-Type\":\"text/html\",\"Connection\":\"keep-alive\"}},\"producer\":{\"modsecurity\":\"ModSecurity v3.0.4 (Linux)\",\"connector\":\"ModSecurity-nginx v1.0.1\",\"secrules_engine\":\"Enabled\",\"components\":[\"OWASP_CRS/3.0.2\\\"\"]},\"messages\":[{\"message\":\"Host header is a numeric IP address\",\"details\":{\"match\":\"Matched \\\"Operator `Rx' with parameter `^[\\\\d.:]+$' against variable `REQUEST_HEADERS:Host' (Value: `192.168.5.187' )\",\"reference\":\"o0,13v65,13\",\"ruleId\":\"920350\",\"file\":\"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf\",\"lineNumber\":\"777\",\"data\":\"192.168.5.187\",\"severity\":\"4\",\"ver\":\"OWASP_CRS/3.0.0\",\"rev\":\"2\",\"tags\":[\"application-multi\",\"language-multi\",\"platform-multi\",\"attack-protocol\",\"OWASP_CRS/PROTOCOL_VIOLATION/IP_HOST\",\"WASCTC/WASC-21\",\"OWASP_TOP_10/A7\",\"PCI/6.5.10\"],\"maturity\":\"9\",\"accuracy\":\"9\"}},{\"message\":\"Path Traversal Attack (/../)\",\"details\":{\"match\":\"Matched \\\"Operator `Rx' with parameter `(?i)(?:\\\\x5c|(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5 (400 characters omitted)' against variable `REQUEST_URI_RAW' (Value: `/aphpfilethatdonotexist.php?=../../etc/passwd' )\",\"reference\":\"o31,4v4,45\",\"ruleId\":\"930100\",\"file\":\"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf\",\"lineNumber\":\"29\",\"data\":\"Matched Data: /../ found within REQUEST_URI_RAW: /aphpfilethatdonotexist.php?=../../etc/passwd\",\"severity\":\"2\",\"ver\":\"OWASP_CRS/3.0.0\",\"rev\":\"3\",\"tags\":[\"application-multi\",\"language-multi\",\"platform-multi\",\"attack-lfi\",\"OWASP_CRS/WEB_ATTACK/DIR_TRAVERSAL\"],\"maturity\":\"9\",\"accuracy\":\"7\"}},{\"message\":\"Path Traversal Attack (/../)\",\"details\":{\"match\":\"Matched \\\"Operator `Pm' with parameter `..\\\\ ../' against variable `REQUEST_URI' (Value: `/aphpfilethatdonotexist.php?=../../etc/passwd' )\",\"reference\":\"o29,3v4,45\",\"ruleId\":\"930110\",\"file\":\"/usr/local/owasp-modsecurity-crs-3.0.2/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf\",\"lineNumber\":\"55\",\"data\":\"Matched Data: ../ found within REQUEST_URI: /aphpfilethatdonotexist.php?=../../etc/passwd\",\"severity\":\"2\",\"ver\":\"OWASP_CRS/3.0.0\",\"rev\":\"1\",\"tags\":[],\"maturity\":\"9\",\"accuracy\":\"7\"}},{\"message\":\"OS File Access Attempt\",\"details\":{\"match\":\"Matched \\\"Operator `PmFromFile' with parameter `lfi-os-files.data' against variable `ARGS:' (Value: `../../etc/passwd' )\",\"reference\":\"o6,10v33,16t:utf8toUnicode,t:urlDecodeUni,t:normalizePathWin,t:lowercase\",\"ruleId\":\"930120\",\"file\":\"/usr/local/owasp-modsecurity-cr\n",
"@timestamp" => 2020-03-31T14:23:49.188Z,
"@version" => "1"
}