Hi,
I've encountered an issue where only a subset of the logs are being processed via ELK stack:
Digesting logs from k8s pods using kubernetes Hints based autodiscover with logstash.
Pod annotations:
annotations:
co.elastic.adm-server-analytics2/enabled: "true"
co.elastic.adm-server-analytics2/json.add_error_key: "true"
co.elastic.adm-server-analytics2/json.keys_under_root: "true"
co.elastic.adm-server-analytics2/json.overwrite_keys: "true"
Document without applying on JSON filter in logstash (Some sensitive data was asterisked) :
{
"_index": "logstash-2020.11.05",
"_type": "_doc",
"_id": "CZX9mHUBhSUPEo1ipqbi",
"_version": 1,
"_score": null,
"_source": {
"@timestamp": "2020-11-05T15:19:17.565Z",
"kubernetes": {
"replicaset": {
"name": "adm-server-analytics2-b585d4f85"
},
"container": {
"image": "********************************",
"name": "adm-server"
},
"node": {
"name": "ip-*****ec2.internal"
},
"pod": {
"uid": "b765b845-5456-415a-bc04-c9b44da80314",
"name": "adm-server-analytics2-b585d4f85-crkct"
},
"labels": {
"release": "adm-server-analytics2",
"pod-template-hash": "b585d4f85",
"app": "adm-server"
},
"namespace": "frontend"
},
"fields": {
"type": "eks-use1-dev-pods"
},
"ecs": {
"version": "1.4.0"
},
"input": {
"type": "container"
},
"host": {
"name": "filebeat-t6ksh"
},
"cloud": {
"availability_zone": "us-east-1d",
"image": {
"id": "ami-******"
},
"instance": {
"id": "i-00b53d8b4d9fe2fa4"
},
"machine": {
"type": "m4.2xlarge"
},
"account": {
"id": "********"
},
"region": "us-east-1",
"provider": "aws"
},
"tags": [
"_grokparsefailure"
],
"agent": {
"type": "filebeat",
"id": "1165387d-1c52-49d5-9067-81468ff21de1",
"hostname": "filebeat-t6ksh",
"version": "7.6.2",
"ephemeral_id": "8927fa29-dca4-4462-bfb4-9a1364899940"
},
"log": {
"offset": 281480,
"file": {
"path": "/var/lib/docker/containers/b43218b5388a6f98071fe36032be25d727c0fbb922b021c0aa1e39b644d2890b/b43218b5388a6f98071fe36032be25d727c0fbb922b021c0aa1e39b644d2890b-json.log"
}
},
"@version": "1",
"stream": "stdout",
"message": "{\"host\":\"adm-server-analytics2-b585d4f85-crkct\",\"application\":\"Semantic Logger\",\"environment\":\"dev\",\"timestamp\":\"2020-11-05T15:19:17.564902Z\",\"level\":\"info\",\"level_index\":2,\"pid\":1045,\"thread\":\"47248600691420\",\"named_tags\":{\"request_id\":\"********************************-3680-4989-a680-969ccf7f2efa\",\"ip\":\"10.1.84.143\",\"thread_id\":47248600691420,\"jsonlogtype\":\"Ruby\"},\"name\":\"Rails\",\"message\":\"Redirected to\",\"payload\":{\"location\":\"********************************\"}}"
},
"fields": {
"@timestamp": [
"2020-11-05T15:19:17.565Z"
]
},
"highlight": {
"kubernetes.pod.name.keyword": [
"@kibana-highlighted-field@adm-server-analytics2-b585d4f85-crkct@/kibana-highlighted-field@"
],
"fields.type.keyword": [
"@kibana-highlighted-field@eks-use1-dev-pods@/kibana-highlighted-field@"
],
"kubernetes.namespace.keyword": [
"@kibana-highlighted-field@frontend@/kibana-highlighted-field@"
]
},
"sort": [
1604589557565
]
}
After adding the following config in logstash-filter.json:
if ([kubernetes][container][name] == "adm-server") {
json {
source => "message"
}
}
logstash stopped creating documents and the following error started to appear:
[2020-11-05T15:22:23,866][WARN ][logstash.filters.json ][main] Error parsing json {:source=>"message", :raw=>"Completed #index", :exception=>#<LogStash::Json::ParserError: Unrecognized token 'Completed': was expecting ('true', 'false' or 'null')
at [Source: (byte[])"Completed #index"; line: 1, column: 11]>}
Tried all possible combinations of annotation, but with little success. Can you please assist?