Parsing multiline events with logstash

[olatunde.tokun (ola)]

Hello all,

I'm trying to parse this multiline log from kafka, its a mix of multilines, json and kv.

Using the multiline codec and grok or KV doesnt seem to produce the expected results. Any thoughts on how I can parse these out fully in the most efficient way would be appreciated.

logstash parser

 /usr/share/logstash/bin/logstash -e '
    input {
        file {
            path => "events.txt"
            codec => multiline { pattern => "^([0-9])\w+$" negate => true what => "previous" }
            start_position => "beginning"
            sincedb_path => "/dev/null"
        }
    }
 filter { 
    grok{
        match => { "message" => [
                "%{DATA:name}\n%{DATA:@timestamp}\n%{DATA:_id}\n%{DATA:_index}\n%{DATA:_type}\n%{GREEDYDATA:others}"
        ]
        }
    }

  }
 output { stdout { codec => rubydebug } }'

Logs to be parsed

0578_notification_of_event

@timestamp: 2019-02-21T17:35:45Z
_id: ninwwwfw5BvD9oPT5jcsJJDWD
_index: security-logging-index
_type: _doc
destination: {
   "domain": "Global \\ Domain",
   "ip": "10.1.10.1",
   "is_proxy": "true",
   "locality": "public",
   "port": 80
}
event: {
   "access_control": {
       "policy": "Internal-Policy",
       "rule": ""
   },
   "action": "Block",
   "category": "URL Block",
   "count": 43883,
   "end": "2019-02-21T17:35:45Z",
   "file_count": 0,
   "ioc_count": 0,
   "ips_count": 0,
   "network_analysis": {
       "policy": ""
   },
   "provider": "connection_events",
   "security_intelligence": {
       "category": "",
       "ip": null
   }
}
http: {
   "response": {
       "status_code": 0
   }
}
network: {
   "application": "HTTP",
   "bytes": {
       "received": 70,
       "sent": 411
   },
   "dns": {
       "hostname_query": "",
       "response": "NoError",
       "ttl": 0
   },
   "icmp": {
       "code": null,
       "received": 1,
       "type": null
   },
   "netbios": {
       "domain": ""
   },
   "netflow": {
       "destination": {
           "mask": 0
       },
       "source": {
           "mask": 0
       }
   },
   "packets": {
       "sent": 3
   },
   "protocol": "tcp",
   "tcp_flags": 0,
   "web_application": null
}
num_hits: 1
num_matches: 1
observer: {
   "hostname": "host.local",
   "ingress": {
       "interface": "eth0",
       "security_zone": "Internal"
   },
   "ip": "192.168.1.2",
   "type": "security-log",
   "vendor": "foo",
   "version": "2.1"
}
source: {
   "application": null,
   "application_version": "",
   "ip": "192.168.1.1",
   "is_proxy": "false",
   "locality": "private",
   "port": 63776
}

url: {
   "category": "",
   "domain": "google.com",
   "full": "http://google.com",
   "path": "443",
   "reputation": "Risk unknown"
}