", "verdict"=>1, "id"=>2167819567973198043, "creationTime"=>1715067533284, "pid"=>10404, "elevationType"=>3}, "endpointId"=>""}}], :response=>{"index"=>{"status"=>400, "error"=>{"type"=>"document_parsing_exception", "reason"=>"[1:5039] failed to parse field [parsed_json.processes.id] of type [long] in document with id 'Fs3-UY8BArD3qwQrDVhY'. Preview of field's value: '17055519350220007344'", "caused_by"=>{"type"=>"x_content_parse_exception", "reason"=>"[1:5059] Numeric value (17055519350220007344) out of range of long (-9223372036854775808 - 9223372036854775807)\n at [Source: (byte[])\"{\"service\":{\"type\":\"logstash\"},\"input\":{\"type\":\"log\"},\"@version\":\"1\",\"agent\":{\"id\":\"f6b16520-a6b8-43a2-8b9d-c06af5bfa8eb\",\"version\":\"8.12.2\",\"ephemeral_id\":\"eea15e70-0bec-44b1-b1a2-2ef38469277e\",\"name\":\"IZDT-226\",\"type\":\"filebeat\"},\"host\":{\"os\":{\"platform\":\"windows\",\"kernel\":\"10.0.11.4291 (WinBuild.160101.0800)\",\"version\":\"10.0\",\"name\":\"Windows 10 Pro\",\"build\":\"19045.4291\",\"type\":\"windows\",\"family\":\"windows\"},\"ip\":[\"fe80::658:99ed:7bfb:613c\",\"10.125.103.10\"],\"mac\":[\"F8-BC-12-q2-AD\"],\"name\"\"[truncated 7761 bytes]; line: 1, column: 5059]", "caused_by"=>{"type"=>"input_coercion_exception", "reason"=>"Numeric value (17055519350220007344) out of range of long (-9223372036854775808 - 9223372036854775807)\n at [Source: (byte[])\"{\"service\":{\"type\":\"logstash\"},\"input\":{\"type\":\"log\"},\"@version\":\"1\",\"agent\":{\"id\":\"f6b16520-a6b8-43a2-8b9d-c06af5bfa8eb\",\"version\":\"8.12.2\",\"ephemeral_id\":\"eea15e70-0bec-44b1-b1a2-2ef38469277e\",\"name\":\"IT-226\",\"type\":\"filebeat\"},\"host\":{\"os\":{\"platform\":\"windows\",\"kernel\":\"10.0.19041.4291 (WinBuild.160101.0800)\",\"version\":\"10.0\",\"name\":\"Windows 10 Pro\",\"build\":\"19045.4291\",\"type\":\"windows\",\"family\":\"windows\"},\"ip\":[\"fe80::658:99ed:7bfb:613c\",\"10.12.1.1\"],\"mac\":[\"F8-BC-12-5D-C2-AD\"],\"name\"\"[truncated 7761 bytes]; line: 1, column: 5059]"}}}}}}
```
But for the below pipeline
cat /etc/logstash/conf.d/openedr.conf
```auto
# Beats -> Logstash -> Elasticsearch pipeline.
input {
beats {
port => 5044
}
}
filter {
json {
source => "event.original"
target => "parsed_event"
}
if [parsed_event] {
mutate {
add_field => { "customer_id" => "%{[parsed_event][customerId]}" }
add_field => { "device_name" => "%{[parsed_event][deviceName]}" }
add_field => { "base_event_type" => "%{[parsed_event][baseEventType]}" }
add_field => { "base_type" => "%{[parsed_event][baseType]}" }
add_field => { "session_user" => "%{[parsed_event][sessionUser]}" }
add_field => { "event_type" => "%{[parsed_event][eventType]}" }
add_field => { "version" => "%{[parsed_event][version]}" }
# Add similar lines for other desired fields
}
}
}
output {
elasticsearch {
hosts => ["https://ip-addr:9200"]
user => "elastic"
password => "passwd"
index => "my_index"
ssl_verification_mode => "full"
ssl_certificate_authorities => "/etc/elasticsearch/certs/http_ca.crt"
ssl_enabled => true
}
}
```
i'm not getting any error log or war log in logstash-plain.log file. And i'm not able to see any logs in kibana\>discover also fields are empty
🥺
---
### Author:  [@sandraimmaculate](https://discuss.elastic.co/u/sandraimmaculate)
#### Post date: [May 7, 2024, 12:20pm UTC](https://discuss.elastic.co/t/the-whole-log-of-openedr-is-inside-the-message-feild-in-kibana-urgent/358535/35 "2024-05-07T12:20:31Z")
```auto
error"=>{"type"=>"document_parsing_exception", "reason"=>"[1:846] failed to parse field [parsed_json.processes.id] of type [long] in document
```
this was the error for the pipeline
```auto
input {
beats {
port => 5044
ecs_compatibility => disabled
}
}
filter {
json {
source => "message"
target => "parsed_json"
remove_field => ["message"]
}
}
output {
your output
}
```
i have changed the datatype of id in mapping from long to keyword
still i'm getting same
do i have to do anything??
---
### Author:  [@leandrojmp](https://discuss.elastic.co/u/leandrojmp)
#### Post date: [May 7, 2024, 12:53pm UTC](https://discuss.elastic.co/t/the-whole-log-of-openedr-is-inside-the-message-feild-in-kibana-urgent/358535/36 "2024-05-07T12:53:53Z")
Did you delete the old index and created a new one? The template is only applied when an index is created.
You need to create a new index.
---
### Author:  [@sandraimmaculate](https://discuss.elastic.co/u/sandraimmaculate)
#### Post date: [May 7, 2024, 12:56pm UTC](https://discuss.elastic.co/t/the-whole-log-of-openedr-is-inside-the-message-feild-in-kibana-urgent/358535/37 "2024-05-07T12:56:22Z")
yes i have created a new index
#index2
```auto
PUT /my_index1
{
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"baseEventType": {
"type": "integer"
},
"baseType": {
"type": "integer"
},
"childProcess": {
"properties": {
"cmdLine": {
"type": "text"
},
"creationTime": {
"type": "date"
},
"elevationType": {
"type": "integer"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "keyword"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"scriptContent": {
"type": "text"
},
"verdict": {
"type": "integer"
}
}
},
"customerId": {
"type": "keyword"
},
"deviceName": {
"type": "keyword"
},
"endpointId": {
"type": "keyword"
},
"eventType": {
"type": "keyword"
},
"processes": {
"type": "nested",
"properties": {
"creationTime": {
"type": "date"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "keyword"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"userName": {
"type": "keyword"
},
"verdict": {
"type": "integer"
}
}
},
"sessionUser": {
"type": "keyword"
},
"time": {
"type": "date"
},
"type": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
}
}
#mapping
PUT /_index_template/my_index_template
{
"index_patterns": ["my_index"],
"data_stream": {},
"template": {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"parsed_json": { // Assuming your fields are nested under a `parsed_json` field
"properties": {
"baseEventType": {
"type": "integer"
},
"baseType": {
"type": "integer"
},
"childProcess": {
"properties": {
"cmdLine": {
"type": "text"
},
"creationTime": {
"type": "date"
},
"elevationType": {
"type": "integer"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "keyword"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"scriptContent": {
"type": "text"
},
"verdict": {
"type": "integer"
}
}
},
"customerId": {
"type": "keyword"
},
"deviceName": {
"type": "keyword"
},
"endpointId": {
"type": "keyword"
},
"eventType": {
"type": "keyword"
},
"processes": {
"type": "nested",
"properties": {
"creationTime": {
"type": "date"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "keyword"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"userName": {
"type": "keyword"
},
"verdict": {
"type": "integer"
}
}
},
"sessionUser": {
"type": "keyword"
},
"time": {
"type": "date"
},
"type": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
}
}
}
}
}
```
---
### Author:  [@leandrojmp](https://discuss.elastic.co/u/leandrojmp)
#### Post date: [May 7, 2024, 1:01pm UTC](https://discuss.elastic.co/t/the-whole-log-of-openedr-is-inside-the-message-feild-in-kibana-urgent/358535/38 "2024-05-07T13:01:23Z")
> [@sandraimmaculate](#):
>
> ` "data_stream": {},`
Remove this from your template, this will complicate things in this moment, use a normal index.
Then use this template:
```auto
PUT /_index_template/my_index_template
{
"index_patterns": ["my_index"],
"template": {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"parsed_json": { // Assuming your fields are nested under a `parsed_json` field
"properties": {
"baseEventType": {
"type": "integer"
},
"baseType": {
"type": "integer"
},
"childProcess": {
"properties": {
"cmdLine": {
"type": "text"
},
"creationTime": {
"type": "date"
},
"elevationType": {
"type": "integer"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "keyword"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"scriptContent": {
"type": "text"
},
"verdict": {
"type": "integer"
}
}
},
"customerId": {
"type": "keyword"
},
"deviceName": {
"type": "keyword"
},
"endpointId": {
"type": "keyword"
},
"eventType": {
"type": "keyword"
},
"processes": {
"type": "nested",
"properties": {
"creationTime": {
"type": "date"
},
"flsVerdict": {
"type": "integer"
},
"id": {
"type": "keyword"
},
"imageHash": {
"type": "keyword"
},
"imagePath": {
"type": "keyword"
},
"pid": {
"type": "integer"
},
"userName": {
"type": "keyword"
},
"verdict": {
"type": "integer"
}
}
},
"sessionUser": {
"type": "keyword"
},
"time": {
"type": "date"
},
"type": {
"type": "keyword"
},
"version": {
"type": "keyword"
}
}
}
}
}
}
}
```
And in your logstash output use `index => my_index`, but before it confirm that `my_index` **does not** exists in your elasticsearch cluster yet.
---
### Author:  [@sandraimmaculate](https://discuss.elastic.co/u/sandraimmaculate)
#### Post date: [May 7, 2024, 1:03pm UTC](https://discuss.elastic.co/t/the-whole-log-of-openedr-is-inside-the-message-feild-in-kibana-urgent/358535/39 "2024-05-07T13:03:31Z")
thank you for you immediate response i will try the above one and let you know
---
### Author:  [@sandraimmaculate](https://discuss.elastic.co/u/sandraimmaculate)
#### Post date: [May 7, 2024, 1:18pm UTC](https://discuss.elastic.co/t/the-whole-log-of-openedr-is-inside-the-message-feild-in-kibana-urgent/358535/40 "2024-05-07T13:18:00Z")
thanks a lot, almost its working i'm able to see the logs parsed correctly in its own fields 😇
but at the same time the whole log entry was inside the event.original field and also in parsed\_json.processes field
what in the sense not all log entry but some log entries are repeated storing the values in event.original and parsed\_json.processess
this is inside parsed\_json.processess field
```auto
"userName": [
"Raghavendra@IT-226"
]
},
{
"flsVerdict": [
1
],
"creationTime": [
"2024-04-29T12:19:33.646Z"
],
"imagePath": [
"C:\\Program Files\\Mozilla Firefox\\firefox.exe"
],
"verdict": [
1
],
"pid": [
14548
],
"id": [
"15755395494097282250"
],
"imageHash": [
"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706"
],
"userName": [
"Raghavendra@IT-226"
]
},
{
"flsVerdict": [
1
],
"creationTime": [
"2024-04-29T12:19:34.882Z"
],
"imagePath": [
"C:\\Program Files\\Mozilla Firefox\\updater.exe"
],
"verdict": [
1
],
"pid": [
14696
],
"id": [
"17932368954266111806"
],
"imageHash": [
"f342d1bbe8b0a941fab99916e0a536851feaa7b6"
],
"userName": [
"Raghavendra@IT-226"
]
},
{
"flsVerdict": [
1
],
"creationTime": [
"2024-04-29T12:19:58.884Z"
],
"imagePath": [
"C:\\Program Files\\Mozilla Firefox\\firefox.exe"
],
"verdict": [
1
],
"pid": [
14184
],
"id": [
"6938495181008955520"
],
"imageHash": [
"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706"
],
"userName": [
"Raghavendra@IT-226"
]
},
{
"flsVerdict": [
1
],
"creationTime": [
"2024-04-29T12:19:59.477Z"
],
"imagePath": [
"C:\\Program Files\\Mozilla Firefox\\firefox.exe"
],
"verdict": [
1
],
"pid": [
10452
],
"id": [
"11984015591290737252"
],
"imageHash": [
"fb3c707fcfafb9190c892e0b7c3bb50a30ffc706"
],
"userName": [
"Raghavendra@IT-226"
]
}
]
```
[Previous page](https://discuss.elastic.co/t/the-whole-log-of-openedr-is-inside-the-message-feild-in-kibana-urgent/358535.md?page=1)