The parsing process is abnormal when json contains 'message'

Elastic Stack Logstash
1

I try to parsing modsecurity json log like this:

{
"transaction": {
"client_ip": "116.30.195.51",
"time_stamp": "Sun Sep 17 01:16:56 2017",
"server_id": "d4425e970f21aa46cba26e2f672308813dcda22d",
"client_port": 63083,
"host_ip": "116.30.195.51",
"host_port": 443,
"id": "150558221674.007216",
"request": {
  "method": "GET",
  "http_version": 2.0,
  "uri": "/?../../passwd",
  "headers": {
    "host": "enginx.net",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36",
    "cookie": "wp-settings-1=editor%3Dtinymce%26libraryContent%3Dbrowse%26edit_element_vcUIPanelWidth%3D650%26edit_element_vcUIPanelLeft%3D560px%26edit_element_vcUIPanelTop%3D112px%26advImgDetails%3Dshow%26hidetb%3D1%26post_dfw%3Doff%26urlbutton%3Dfile%26template_window_vcUIPanelWidth%3D1152%26template_window_vcUIPanelLeft%3D144px%26template_window_vcUIPanelTop%3D74px%26imgsize%3Dfull%26mfold%3Do%26posts_list_mode%3Dlist; wp-settings-time-1=1501673287; _ga=GA1.2.102219144.1492348731; _gid=GA1.2.488031018.1505489689; wordfence_verifiedHuman=f4d45d3f2a5b9eb9d1e64fe4622b9d61",
    "upgrade-insecure-requests": "1",
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "accept-encoding": "gzip, deflate, br",
    "accept-language": "zh-CN,zh;q=0.8,zh-TW;q=0.6,en;q=0.4"
  }
},
"response": {
  "http_code": 403,
  "headers": {
    "Server": "nginx",
    "Date": "Sat, 16 Sep 2017 17:16:56 GMT",
    "Content-Type": "text/html",
    "Connection": "close",
    "Strict-Transport-Security": "max-age=31536000; preload; includeSubDomains",
    "Content-Encoding": "gzip"
  }
},
"producer": {
  "modsecurity": "ModSecurity v3.0.0rc1 (Linux)",
  "connector": "ModSecurity-nginx v0.1.1-beta",
  "secrules_engine": "Enabled",
  "components": [
    "OWASP_CRS/3.0.2\""
  ]
},
"messages": [
  {
    "message": "Path Traversal Attack (/../)",
    "details": {
      "match": "Matched \"Operator `Rx' with parameter `(?i)(?:\\x5c|(?:%(?:c(?:0%(?:[2aq]f|5c|9v)|1%(?:[19p]c|8s|af))|2(?:5(?:c(?:0%25af|1%259c)|2f|5c)|%46|f)|(?:(?:f(?:8%8)?0%8|e)0%80%a|bg%q)f|%3(?:2(?:%(?:%6|4)6|F)|5%%63)|u(?:221[56]|002f|EFC8|F025)|1u|5 (400 characters omitted)' against variable `REQUEST_URI_RAW' (Value: `/?../../passwd' )",
      "reference": "o0,5v4,14",
      "ruleId": "930100",
      "file": "/root/codex/owasp-modsecurity-crs/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf",
      "lineNumber": "17",
      "data": "Matched Data: /?../ found within REQUEST_URI_RAW: /?../../passwd",
      "severity": "2",
      "ver": "OWASP_CRS/3.0.0",
      "rev": "3",
      "tags": [
        "application-multi",
        "language-multi",
        "platform-multi",
        "attack-lfi",
        "OWASP_CRS/WEB_ATTACK/DIR_TRAVERSAL"
      ],
      "maturity": "9",
      "accuracy": "7"
    }
  },
  {
    "message": "Path Traversal Attack (/../)",
    "details": {
      "match": "Matched \"Operator `Pm' with parameter `..\\ ../' against variable `REQUEST_URI' (Value: `/?../../passwd' )",
      "reference": "o2,3v4,14t:utf8toUnicode,t:urlDecodeUni,t:removeNulls,t:cmdLine",
      "ruleId": "930110",
      "file": "/root/codex/owasp-modsecurity-crs/rules/REQUEST-930-APPLICATION-ATTACK-LFI.conf",
      "lineNumber": "49",
      "data": "Matched Data: ../ found within REQUEST_URI: /?../../passwd",
      "severity": "2",
      "ver": "OWASP_CRS/3.0.0",
      "rev": "1",
      "tags": [
        "application-multi",
        "language-multi",
        "platform-multi",
        "attack-lfi",
        "OWASP_CRS/WEB_ATTACK/DIR_TRAVERSAL"
      ],
      "maturity": "9",
      "accuracy": "7"
    }
  },
  {
    "message": "Inbound Anomaly Score Exceeded (Total Score: 10)",
    "details": {
      "match": "Matched \"Operator `Ge' with parameter `%{tx.inbound_anomaly_score_threshold}' against variable `TX:ANOMALY_SCORE' (Value: `10' )",
      "reference": "",
      "ruleId": "949110",
      "file": "/root/codex/owasp-modsecurity-crs/rules/REQUEST-949-BLOCKING-EVALUATION.conf",
      "lineNumber": "36",
      "data": "",
      "severity": "2",
      "ver": "",
      "rev": "",
      "tags": [
        "application-multi",
        "language-multi",
        "platform-multi",
        "attack-generic"
      ],
      "maturity": "0",
      "accuracy": "0"
    }
  },
  {
    "message": "Inbound Anomaly Score Exceeded (Total Inbound Score: 10 - SQLI=0,XSS=0,RFI=0,LFI=10,RCE=0,PHPI=0,HTTP=0,SESS=0): Path Traversal Attack (/../)",
    "details": {
      "match": "Matched \"Operator `Ge' with parameter `%{tx.inbound_anomaly_score_threshold}' against variable `TX:INBOUND_ANOMALY_SCORE' (Value: `10' )",
      "reference": "",
      "ruleId": "980130",
      "file": "/root/codex/owasp-modsecurity-crs/rules/RESPONSE-980-CORRELATION.conf",
      "lineNumber": "61",
      "data": "",
      "severity": "0",
      "ver": "",
      "rev": "",
      "tags": [
        "event-correlation"
      ],
      "maturity": "0",
      "accuracy": "0"
    }
  }
]
}
}

as you can see,there have some 'message' string in the log.

when logstash parse these string,It is abnormal:

message
TIM截图20170917021327.png1542×978 46.1 KB

this is my filter:

filter {

if "modsec_audit_log" in [tags] {
    grok {
        match => ["message", "%{GREEDYDATA:temp_msg}"]
    }

    mutate {
        gsub => ["temp_msg", "message","msg"]
    }

    json {
        source => "temp_msg"
        target => "modsec"
    }

    mutate {
        remove_field => [ "temp_msg" ]
    }

    geoip {
        source => "[modsec][transaction][client_ip]"
    }

    date {
        match => [ "[modsec][transaction][time_stamp]" , "EEE MMM dd HH:mm:ss yyyy" ]
    }

    useragent {
        source => "[modsec][transaction][request][headers][user-agent]"
        target => "ua"
    }

}
}

I try to replace the word 'message' to 'msg' before parse by json plugin,but not working.

How can i fix this issues?

2

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.