Filebeat 6.2
I am using filebeat to pick up a file from osquery. The file contains windows events from WEL.
Sample data:
{
"_index": "win_osquery-2018.05.02",
"_type": "doc",
"_id": "5xRGImMBm7muvmysFsee",
"_version": 1,
"_score": null,
"_source": {
"tags": [
"osquery_windows",
"beats_input_raw_event"
],
"counter": "0",
"unixTime": "1525288272",
"type": "win_osquery",
"calendarTime": "Wed May 2 19:11:12 2018 UTC",
"name": "windows_events",
"columns": {
"provider_guid": "{5770385F-C22A-43E0-BF4C-06F5698FFBD9}",
"data": "{\"EventData\":{\"<xmlattr>\":\"\",\"CallTrace\":\"C:\\\\WINDOWS\\\\SYSTEM32\\\\ntdll.dll+a6594|C:\\\\WINDOWS\\\\System32\\\\KERNELBASE.dll+20edd|C:\\\\WINDOWS\\\\system32\\\\wbem\\\\cimwin32.dll+6fb3|C:\\\\WINDOWS\\\\system32\\\\wbem\\\\cimwin32.dll+7471|C:\\\\WINDOWS\\\\SYSTEM32\\\\framedynos.dll+5899|C:\\\\WINDOWS\\\\SYSTEM32\\\\framedynos.dll+adc4|C:\\\\WINDOWS\\\\system32\\\\wbem\\\\wmiprvse.exe+aaf1|C:\\\\WINDOWS\\\\system32\\\\wbem\\\\wmiprvse.exe+a704|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+77de3|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+443ef|C:\\\\WINDOWS\\\\System32\\\\combase.dll+3b00|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+a92b|C:\\\\WINDOWS\\\\System32\\\\combase.dll+967bc|C:\\\\WINDOWS\\\\System32\\\\combase.dll+96e02|C:\\\\WINDOWS\\\\System32\\\\combase.dll+ae8b8|C:\\\\WINDOWS\\\\System32\\\\combase.dll+ac81d|C:\\\\WINDOWS\\\\System32\\\\combase.dll+aaf74|C:\\\\WINDOWS\\\\System32\\\\combase.dll+aa1fc|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+5a194|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+590ad|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+59bfe|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+39927|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+39f7c|C:\\\\WINDOWS\\\\System32\\\\RPCRT4.dll+5426c\",\"GrantedAccess\":\"0x1410\",\"Name\":\"UtcTime,SourceProcessGUID,SourceProcessId,SourceThreadId,SourceImage,TargetProcessGUID,TargetProcessId,TargetImage,GrantedAccess,CallTrace\",\"SourceImage\":\"C:\\\\WINDOWS\\\\system32\\\\wbem\\\\wmiprvse.exe\",\"SourceProcessGUID\":\"{AD35859D-6702-5A8D-0000-00103B690200}\",\"SourceProcessId\":\"2832\",\"SourceThreadId\":\"30988\",\"TargetImage\":\"C:\\\\WINDOWS\\\\system32\\\\lsass.exe\",\"TargetProcessGUID\":\"{AD35859D-66FC-5A8D-0000-0010F4A50000}\",\"TargetProcessId\":\"684\",\"UtcTime\":\"2018-05-02 19:11:05.020\"}}\\x0A",
"eventid": "10",
"source": "Microsoft-Windows-Sysmon/Operational",
"datetime": "2018-05-02T19:11:05.027156300Z",
"time": "1525288265",
"keywords": "-1",
"provider_name": "Microsoft-Windows-Sysmon",
"task": "10",
"level": "4"
},
"epoch": "0",
"hostIdentifier": "HOSTNAME",
"@version": "1",
"@timestamp": "2018-05-02T19:11:19.145Z",
"decorations": {
"os_name": "Microsoft Windows 10 Enterprise",
"username": "USERNAME",
"os_version": "10.0.14393",
"host_uuid": "BD804D56-9368-07D4-B37E-FB9038ABD62D"
},
"offset": 17676584,
"action": "added"
},
"fields": {
"@timestamp": [
"2018-05-02T19:11:19.145Z"
],
"columns.datetime": [
"2018-05-02T19:11:05.027Z"
]
},
"sort": [
1525288279145
]
}
How can I parse the columns.data field for value EventData. This data as a blob is pretty useless but if filebeat could parse this and like it does columns it would make the data much more useful.
filebeat.conf:
---
filebeat.prospectors:
- type: log
enabled: true
json.keys_under_root: true
json.add_error_key: true
paths:
- /path/to/file.log
tags: ["tags1","tags2"]
processors:
- drop_fields:
fields: ["fields","to","drop"]
- decod_json_fields:
fields: ["columns.data"]
process_array: true
max_depth: 6
overwrite_keys: true
output.logstash:
hosts: ["host.com:80]
loadbalance: true
What am I missing. Is it not possible for filebeat to parse the nested json fields?