i have created a new index
PUT /my_index
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"baseEventType": {
"type": "integer"
},
"baseType": {
"type": "integer"
},
"childProcess": {
"properties": {
"cmdLine": {
"type": "text"
},
"creationTime": {
"type": "date"
},
"elevationType": {
"type": "integer"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "long"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"scriptContent": {
"type": "text"
},
"verdict": {
"type": "integer"
}
}
},
"customerId": {
"type": "keyword"
},
"deviceName": {
"type": "keyword"
},
"endpointId": {
"type": "keyword"
},
"eventType": {
"type": "keyword"
},
"processes": {
"type": "nested",
"properties": {
"creationTime": {
"type": "date"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "long"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"userName": {
"type": "keyword"
},
"verdict": {
"type": "integer"
}
}
},
"sessionUser": {
"type": "keyword"
},
"time": {
"type": "date"
},
"type": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
}
}
with parsed_json
PUT /_index_template/my_index_template
{
"index_patterns": ["my_index"],
"data_stream": {},
"template": {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"parsed_json": { // Assuming your fields are nested under a `parsed_json` field
"properties": {
"baseEventType": {
"type": "integer"
},
"baseType": {
"type": "integer"
},
"childProcess": {
"properties": {
"cmdLine": {
"type": "text"
},
"creationTime": {
"type": "date"
},
"elevationType": {
"type": "integer"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "long"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"scriptContent": {
"type": "text"
},
"verdict": {
"type": "integer"
}
}
},
"customerId": {
"type": "keyword"
},
"deviceName": {
"type": "keyword"
},
"endpointId": {
"type": "keyword"
},
"eventType": {
"type": "keyword"
},
"processes": {
"type": "nested",
"properties": {
"creationTime": {
"type": "date"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "long"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"userName": {
"type": "keyword"
},
"verdict": {
"type": "integer"
}
}
},
"sessionUser": {
"type": "keyword"
},
"time": {
"type": "date"
},
"type": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
}
}
}
}
}
now it has all the required fields what it is needed
here is my pipeline
cat openedr.conf
input {
beats {
port => 5044
ecs_compatibility => disabled
}
}
filter {
json {
source => "message"
target => "parsed_json"
remove_field => ["message"]
}
}
output {
elasticsearch {
hosts => ["https://ip-address:9200"]
user => "elastic"
password => "passwd"
index => "my_index"
ssl_verification_mode => "full"
ssl_certificate_authorities => "/etc/elasticsearch/certs/http_ca.crt"
ssl_enabled => true
}
}
im not able to see logs in kibana > discover for that index all feilds are empty
[2024-05-07T13:09:15,170][WARN ][logstash.outputs.elasticsearch][main][6052cd2c09614adc099a18352c4307efc1ca2c35ffbf7b7ea6a659325ffabf67] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"my_index", :routing=>nil}, {"service"=>{"type"=>"logstash"}, "input"=>{"type"=>"log"}, "@version"=>"1", "agent"=>{"id"=>"f6b16520-a6b8-43a2-8b9d-c06af5bfa8eb", "version"=>"8.12.2", "ephemeral_id"=>"eea15e70-0bec-44b1-b1a2-2ef38469277e", "name"=>"IZDT-226", "type"=>"filebeat"}, "host"=>{"os"=>{"platform"=>"windows", "kernel"=>"10.0.19041.4291 (WinBuild.160101.0800)", "version"=>"10.0", "name"=>"Windows 10 Pro", "build"=>"19045.4291", "type"=>"windows", "family"=>"windows"}, "ip"=>["fe80::658:99ed:7bfb:613c", "10.125.1.0"], "mac"=>["F8-BC-12-AD"], "name"=>"izdt-226", "hostname"=>"it-226", "id"=>"b6f1b6a5-2801-46b5-820d-369b087e23fb", "architecture"=>"x86_64"}, "@timestamp"=>2024-05-07T07:38:57.247Z, "ecs"=>{"version"=>"1.12.0"}, "log"=>{"offset"=>1264614, "file"=>{"path"=>"C:\\ProgramData\\edrsvc\\log\\output_events\\2024-05-07.log"}}, "fileset"=>{"name"=>"log"}, "event"=>{"module"=>"logstash", "original"=>"{\"baseEventType\":1,\"baseType\":1,\"childProcess\":{\"cmdLine\":\"\\\"C:\\\\Program Files\\\\Mozilla Firefox\\\\firefox.exe\\\" -contentproc --channel=7860 -childID 3394 -isForBrowser -prefsHandle 9252 -prefMapHandle 9356 -prefsLen 32640 -prefMapSize 250043 -jsInitHandle 1212 -jsInitLen 234952 -parentBuildID 20240419144423 -win32kLockedDown -appDir \\\"C:\\\\Program Files\\\\Mozilla Firefox\\\\browser\\\" - {383ed037-1e9c-4766-a2b9-6ab6318934cc} 10452 \\\"\\\\\\\\.\\\\pipe\\\\gecko-crash-server-pipe.10452\\\" tab /prefetch:2\",\"creationTime\":1715067533284,\"elevationType\":3,\"flsVerdict\":1,\"id\":2167819567973198043,\"imageHash\":\"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706\",\"imagePath\":\"C:\\\\Program Files\\\\Mozilla Firefox\\\\firefox.exe\",\"pid\":10404,\"scriptContent\":\"<undefined>\",\"verdict\":1},\"customerId\":\"\",\"deviceName\":\"IT-226\",\"endpointId\":\"\",\"eventType\":null,\"processes\":[{\"creationTime\":1714392551294,\"flsVerdict\":3,\"id\":17055519350220007344,\"imageHash\":\"\",\"imagePath\":\"System\",\"pid\":4,\"userName\":\"SYSTEM@NT AUTHORITY\",\"verdict\":1},{\"creationTime\":1714392551296,\"flsVerdict\":3,\"id\":10484085230716227874,\"imageHash\":\"fc4b6f48a8a7ece73f1da2370d7b488820f10b0d\",\"imagePath\":\"C:\\\\Windows\\\\System32\\\\smss.exe\",\"pid\":512,\"userName\":\"SYSTEM@NT AUTHORITY\",\"verdict\":1},{\"creationTime\":1714393113631,\"flsVerdict\":3,\"id\":13679504117710993514,\"imageHash\":\"fc4b6f48a8a7ece73f1da2370d7b488820f10b0d\",\"imagePath\":\"C:\\\\Windows\\\\System32\\\\smss.exe\",\"pid\":9732,\"userName\":\"SYSTEM@NT AUTHORITY\",\"verdict\":1},{\"creationTime\":1714393113709,\"flsVerdict\":3,\"id\":13554967919231126822,\"imageHash\":\"875d495de7b1d41eee789f92ff1aaae2495b1419\",\"imagePath\":\"C:\\\\Windows\\\\System32\\\\winlogon.exe\",\"pid\":1652,\"userName\":\"SYSTEM@NT AUTHORITY\",\"verdict\":1},{\"creationTime\":1714393134549,\"flsVerdict\":3,\"id\":4120525655557887168,\"imageHash\":\"e5a153af6ab857fc4752a2bb008b435add5bf0b8\",\"imagePath\":\"C:\\\\Windows\\\\System32\\\\userinit.exe\",\"pid\":8084,\"userName\":\"Raghavendra@IZDT-226\",\"verdict\":1},{\"creationTime\":1714393134645,\"flsVerdict\":3,\"id\":8323067076655652240,\"imageHash\":\"61ee53287d7aa2abbf323cc04e4475ae07ed6e75\",\"imagePath\":\"C:\\\\Windows\\\\explorer.exe\",\"pid\":8256,\"userName\":\"Raghavendra@IZDT-226\",\"verdict\":1},{\"creationTime\":1714393173371,\"flsVerdict\":1,\"id\":993482044620317102,\"imageHash\":\"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706\",\"imagePath\":\"C:\\\\Program Files\\\\Mozilla Firefox\\\\firefox.exe\",\"pid\":14524,\"userName\":\"Raghavendra@IZDT-226\",\"verdict\":1},{\"creationTime\":1714393173646,\"flsVerdict\":1,\"id\":15755395494097282250,\"imageHash\":\"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706\",\"imagePath\":\"C:\\\\Program Files\\\\Mozilla Firefox\\\\firefox.exe\",\"pid\":14548,\"userName\":\"Raghavendra@IZDT-226\",\"verdict\":1},{\"creationTime\":1714393174882,\"flsVerdict\":1,\"id\":17932368954266111806,\"imageHash\":\"f342d1bbe8b0a941fab99916e0a536851feaa7b6\",\"imagePath\":\"C:\\\{"imageHash"=>"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706", "verdict"=>1, "imagePath"=>"C:\\Program Files\\Mozilla Firefox\\firefox.exe", "id"=>15755395494097282250, "creationTime"=>1714393173646, "pid"=>14548, "userName"=>"Raghavendra@IT-226", "flsVerdict"=>1}, {"imageHash"=>"f342d1bbe8b0a941fab99916e0a536851feaa7b6", "verdict"=>1, "imagePath"=>"C:\\Program Files\\Mozilla Firefox\\updater.exe", "id"=>17932368954266111806, "creationTime"=>1714393174882, "pid"=>14696, "userName"=>"Raghavendra@It-226", "flsVerdict"=>1}, {"imageHash"=>"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706", "verdict"=>1, "imagePath"=>"C:\\Program Files\\Mozilla Firefox\\firefox.exe", "id"=>6938495181008955520, "creationTime"=>1714393198884, "pid"=>14184, "userName"=>"Raghavendra@IT-226", "flsVerdict"=>1}, {"imageHash"=>"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706", "verdict"=>1, "imagePath"=>"C:\\Program Files\\Mozilla Firefox\\firefox.exe", "id"=>11984015591290737252, "creationTime"=>1714393199477, "pid"=>10452, "userName"=>"Raghavendra@IT-226", "flsVerdict"=>1}], "childProcess"=>{"flsVerdict"=>1, "cmdLine"=>"\"C:\\Program Files\\Mozilla Firefox\\firefox.exe\" -contentproc --channel=7860 -childID 3394 -isForBrowser -prefsHandle 9252 -prefMapHandle 9356 -prefsLen 32640 -prefMapSize 250043 -jsInitHandle 1212 -jsInitLen 234952 -parentBuildID 20240419144423 -win32kLockedDown -appDir \"C:\\Program Files\\Mozilla Firefox\\browser\" - {383ed037-1e9c-4766-a2b9-6ab6318934cc} 10452 \"\\\\.\\pipe\\gecko-crash-server-pipe.10452\" tab /prefetch:2", "imageHash"=>"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706", "imagePath"=>"C:\\Program Files\\Mozilla Firefox\\firefox.exe", "scriptContent"=>"<undefined>", "verdict"=>1, "id"=>2167819567973198043, "creationTime"=>1715067533284, "pid"=>10404, "elevationType"=>3}, "endpointId"=>""}}], :response=>{"index"=>{"status"=>400, "error"=>{"type"=>"document_parsing_exception", "reason"=>"[1:5039] failed to parse field [parsed_json.processes.id] of type [long] in document with id 'Fs3-UY8BArD3qwQrDVhY'. Preview of field's value: '17055519350220007344'", "caused_by"=>{"type"=>"x_content_parse_exception", "reason"=>"[1:5059] Numeric value (17055519350220007344) out of range of long (-9223372036854775808 - 9223372036854775807)\n at [Source: (byte[])\"{\"service\":{\"type\":\"logstash\"},\"input\":{\"type\":\"log\"},\"@version\":\"1\",\"agent\":{\"id\":\"f6b16520-a6b8-43a2-8b9d-c06af5bfa8eb\",\"version\":\"8.12.2\",\"ephemeral_id\":\"eea15e70-0bec-44b1-b1a2-2ef38469277e\",\"name\":\"IZDT-226\",\"type\":\"filebeat\"},\"host\":{\"os\":{\"platform\":\"windows\",\"kernel\":\"10.0.11.4291 (WinBuild.160101.0800)\",\"version\":\"10.0\",\"name\":\"Windows 10 Pro\",\"build\":\"19045.4291\",\"type\":\"windows\",\"family\":\"windows\"},\"ip\":[\"fe80::658:99ed:7bfb:613c\",\"10.125.103.10\"],\"mac\":[\"F8-BC-12-q2-AD\"],\"name\"\"[truncated 7761 bytes]; line: 1, column: 5059]", "caused_by"=>{"type"=>"input_coercion_exception", "reason"=>"Numeric value (17055519350220007344) out of range of long (-9223372036854775808 - 9223372036854775807)\n at [Source: (byte[])\"{\"service\":{\"type\":\"logstash\"},\"input\":{\"type\":\"log\"},\"@version\":\"1\",\"agent\":{\"id\":\"f6b16520-a6b8-43a2-8b9d-c06af5bfa8eb\",\"version\":\"8.12.2\",\"ephemeral_id\":\"eea15e70-0bec-44b1-b1a2-2ef38469277e\",\"name\":\"IT-226\",\"type\":\"filebeat\"},\"host\":{\"os\":{\"platform\":\"windows\",\"kernel\":\"10.0.19041.4291 (WinBuild.160101.0800)\",\"version\":\"10.0\",\"name\":\"Windows 10 Pro\",\"build\":\"19045.4291\",\"type\":\"windows\",\"family\":\"windows\"},\"ip\":[\"fe80::658:99ed:7bfb:613c\",\"10.12.1.1\"],\"mac\":[\"F8-BC-12-5D-C2-AD\"],\"name\"\"[truncated 7761 bytes]; line: 1, column: 5059]"}}}}}}
But for the below pipeline
cat /etc/logstash/conf.d/openedr.conf
# Beats -> Logstash -> Elasticsearch pipeline.
input {
beats {
port => 5044
}
}
filter {
json {
source => "event.original"
target => "parsed_event"
}
if [parsed_event] {
mutate {
add_field => { "customer_id" => "%{[parsed_event][customerId]}" }
add_field => { "device_name" => "%{[parsed_event][deviceName]}" }
add_field => { "base_event_type" => "%{[parsed_event][baseEventType]}" }
add_field => { "base_type" => "%{[parsed_event][baseType]}" }
add_field => { "session_user" => "%{[parsed_event][sessionUser]}" }
add_field => { "event_type" => "%{[parsed_event][eventType]}" }
add_field => { "version" => "%{[parsed_event][version]}" }
# Add similar lines for other desired fields
}
}
}
output {
elasticsearch {
hosts => ["https://ip-addr:9200"]
user => "elastic"
password => "passwd"
index => "my_index"
ssl_verification_mode => "full"
ssl_certificate_authorities => "/etc/elasticsearch/certs/http_ca.crt"
ssl_enabled => true
}
}
i'm not getting any error log or war log in logstash-plain.log file. And i'm not able to see any logs in kibana>discover also fields are empty
