Hello all,
I'm trying to parse this multiline log from kafka, its a mix of multilines, json and kv.
Using the multiline codec and grok or KV doesnt seem to produce the expected results. Any thoughts on how I can parse these out fully in the most efficient way would be appreciated.
logstash parser
/usr/share/logstash/bin/logstash -e '
input {
file {
path => "events.txt"
codec => multiline { pattern => "^([0-9])\w+$" negate => true what => "previous" }
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
grok{
match => { "message" => [
"%{DATA:name}\n%{DATA:@timestamp}\n%{DATA:_id}\n%{DATA:_index}\n%{DATA:_type}\n%{GREEDYDATA:others}"
]
}
}
}
output { stdout { codec => rubydebug } }'
Logs to be parsed
0578_notification_of_event
@timestamp: 2019-02-21T17:35:45Z
_id: ninwwwfw5BvD9oPT5jcsJJDWD
_index: security-logging-index
_type: _doc
destination: {
"domain": "Global \\ Domain",
"ip": "10.1.10.1",
"is_proxy": "true",
"locality": "public",
"port": 80
}
event: {
"access_control": {
"policy": "Internal-Policy",
"rule": ""
},
"action": "Block",
"category": "URL Block",
"count": 43883,
"end": "2019-02-21T17:35:45Z",
"file_count": 0,
"ioc_count": 0,
"ips_count": 0,
"network_analysis": {
"policy": ""
},
"provider": "connection_events",
"security_intelligence": {
"category": "",
"ip": null
}
}
http: {
"response": {
"status_code": 0
}
}
network: {
"application": "HTTP",
"bytes": {
"received": 70,
"sent": 411
},
"dns": {
"hostname_query": "",
"response": "NoError",
"ttl": 0
},
"icmp": {
"code": null,
"received": 1,
"type": null
},
"netbios": {
"domain": ""
},
"netflow": {
"destination": {
"mask": 0
},
"source": {
"mask": 0
}
},
"packets": {
"sent": 3
},
"protocol": "tcp",
"tcp_flags": 0,
"web_application": null
}
num_hits: 1
num_matches: 1
observer: {
"hostname": "host.local",
"ingress": {
"interface": "eth0",
"security_zone": "Internal"
},
"ip": "192.168.1.2",
"type": "security-log",
"vendor": "foo",
"version": "2.1"
}
source: {
"application": null,
"application_version": "",
"ip": "192.168.1.1",
"is_proxy": "false",
"locality": "private",
"port": 63776
}
url: {
"category": "",
"domain": "google.com",
"full": "http://google.com",
"path": "443",
"reputation": "Risk unknown"
}