Modsecurity log (split on audit_data[messages]) Only String and Array types are splittable

Elastic Stack Logstash
1 
{
    "transaction": {
        "time": "20/Jan/2024:00:10:51 +0530",
        "transaction_id": "16717361827536742843",
        "remote_address": "20.1.198.110",
        "remote_port": 80,
        "local_address": "127.0.0.1",
        "local_port": 80
    },
    "request": {
        "request_line": "GET /.env HTTP/1.1",
        "headers": {
            "Connection": "keep-alive",
            "Accept": "*/*",
            "Accept-Encoding": "gzip, deflate",
            "Host": "94.217.32.168",
            "User-Agent": "Mozilla/5.0 (Linux; U; Android 4.4.2; en-US; HM NOTE 1W Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 UCBrowser/11.0.5.850 U3/0.8.0 Mobile Safari/534.30"
        }
    },
    "response": {
        "protocol": "HTTP/1.1",
        "status": 0,
        "headers": {}
    },
    "audit_data": {
        "messages": [
            "Warning. Pattern match ",
            "Warning. Matched phrase ",
            "Warning. Operator GE matched 5 "
        ],
        "action": {
            "intercepted": true,
            "phase": 2,
            "message": "Operator GE matched 5 at TX:anomaly_score."
        },
        "handler": "IIS",
        "stopwatch": {
            "p1": 2004,
            "p2": 2005,
            "p3": 0,
            "p4": 0,
            "p5": 1003,
            "sr": 2004,
            "sw": 0,
            "l": 0,
            "gc": 0
        },
        "producer": [
            "ModSecurity for IIS (STABLE)/2.9.3 (http://www.modsecurity.org/)",
            "OWASP_CRS/3.0.2"
        ],
        "server": "ModSecurity Standalone",
        "engine_mode": "ENABLED"
    }
}

logstash.conf

input {
  file{
    path => "C:/inetpub/logs/modsec_audit.log"
    start_position => "beginning"
    sincedb_path => "NUL"
    codec => json
  }
}
filter {
  mutate { remove_field => ["[event][original]"] }

  mutate { remove_field => ["[request][headers][User-Agent]"] }

  if [response][body] { 
    mutate { remove_field => ["[response][body]"] }
  }
  split { field => "[audit_data][messages]"}
}
output {
 file {
   codec => json
   path => "c:/temp/logstash_out.log"
 }
}

But got error

Only String and Array types are splittable. field:[audit_data][messages] is of type = NilClass

All I want is multiple mesages (with all other field intact) with each [audit_data][messages]

What's wrong ?

2

Most likely there are lines in the file that do not contain the [audit_data][messages] field. Make the split conditional

if [audit_data][messages] { split { field => "[audit_data][messages]"} }
3

I copied that one single json in seperate file and test it. Getting same error. Is this a bug ?

4

I don't think so.

input { generator { count => 1 lines => [ '{ "transaction": { "time": "20/Jan/2024:00:10:51 +0530", "transaction_id": "16717361827536742843", "remote_address": "20.1.198.110", "remote_port": 80, "local_address": "127.0.0.1", "local_port": 80 }, "request": { "request_line": "GET /.env HTTP/1.1", "headers": { "Connection": "keep-alive", "Accept": "*/*", "Accept-Encoding": "gzip, deflate", "Host": "94.217.32.168", "User-Agent": "Mozilla/5.0 (Linux; U; Android 4.4.2; en-US; HM NOTE 1W Build/KOT49H) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 UCBrowser/11.0.5.850 U3/0.8.0 Mobile Safari/534.30" } }, "response": { "protocol": "HTTP/1.1", "status": 0, "headers": {} }, "audit_data": { "messages": [ "Warning. Pattern match ", "Warning. Matched phrase ", "Warning. Operator GE matched 5 " ], "action": { "intercepted": true, "phase": 2, "message": "Operator GE matched 5 at TX:anomaly_score." }, "handler": "IIS", "stopwatch": { "p1": 2004, "p2": 2005, "p3": 0, "p4": 0, "p5": 1003, "sr": 2004, "sw": 0, "l": 0, "gc": 0 }, "producer": [ "ModSecurity for IIS (STABLE)/2.9.3 (http://www.modsecurity.org/)", "OWASP_CRS/3.0.2" ], "server": "ModSecurity Standalone", "engine_mode": "ENABLED" } }' ] } }
output { stdout { codec => rubydebug { metadata => false } } }
filter {
    json { source => "message" remove_field => [ "message" ] }
    mutate { remove_field => [ "[event][original]", "[request][headers][User-Agent]", "[response][body]" ] }
    split { field => "[audit_data][messages]"}
}

produces three events. To use a json codec your JSON needs to be on a single line. If it is pretty-printed then you need to use a multiline codec and a json filter.

5

To use a json codec your JSON needs to be on a single line.

Yes. That solved the issue.

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.